From 2ca72c8bda872e0332f1fa3169805445754cda8e Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Wed, 10 Mar 2021 10:05:24 -0800 Subject: [PATCH 01/32] Improve publishing performance --- eng/publishing/v3/publish-assets.yml | 24 + eng/publishing/v3/setup-maestro-vars.yml | 2 + .../SdkTasks/PublishArtifactsInManifest.proj | 8 +- .../src/BuildArtifacts.cs | 35 + .../src/PublishArtifactsInManifest.cs | 19 +- .../src/PublishArtifactsInManifestBase.cs | 671 +++++++++++++++++- .../src/PublishArtifactsInManifestV3.cs | 80 +-- 7 files changed, 753 insertions(+), 86 deletions(-) create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/src/BuildArtifacts.cs diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml index d85ed016963..798ce54fe70 100644 --- a/eng/publishing/v3/publish-assets.yml +++ b/eng/publishing/v3/publish-assets.yml @@ -22,6 +22,8 @@ jobs: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - name: AzDOBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + - name: AzDOAccount + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildAccount'] ] pool: vmImage: 'windows-2019' steps: @@ -30,6 +32,7 @@ jobs: continueOnError: true inputs: buildType: specific + enabled: false buildVersionToDownload: specific project: $(AzDOProjectName) pipeline: $(AzDOPipelineId) @@ -42,6 +45,22 @@ jobs: PdbArtifacts/** downloadPath: '$(Build.ArtifactStagingDirectory)' + - task: DownloadBuildArtifacts@0 + displayName: Download Build Assets + continueOnError: true + enabled: true + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + downloadType: 'specific' + itemPattern: | + AssetManifests/** + PdbArtifacts/** + downloadPath: '$(Build.ArtifactStagingDirectory)' + - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' @@ -86,6 +105,11 @@ jobs: /p:MsdlToken=$(microsoft-symbol-server-pat) /p:SymWebToken=$(symweb-symbol-server-pat) /p:BuildQuality='${{ parameters.buildQuality }}' + /p:AzdoApiToken='$(dn-bot-dotnet-build-rw-code-rw)' + /p:TemporaryStagingDir='$(Build.ArtifactStagingDirectory)/' + /p:BuildId='$(AzDOBuildId)' + /p:AzureDevOpsOrg='dnceng' + /p:AzureProject='$(AzDOProjectName)' - template: /eng/common/templates/steps/publish-logs.yml parameters: diff --git a/eng/publishing/v3/setup-maestro-vars.yml b/eng/publishing/v3/setup-maestro-vars.yml index 804e9faee96..ae2e65527b6 100644 --- a/eng/publishing/v3/setup-maestro-vars.yml +++ b/eng/publishing/v3/setup-maestro-vars.yml @@ -38,6 +38,7 @@ jobs: $AzureDevOpsProject = $buildInfo.azureDevOpsProject $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + $AzureDevOpsAccount = $buildInfo.azureDevOpsAccount Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels" @@ -46,6 +47,7 @@ jobs: Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject" Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId" Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId" + Write-Host "##vso[task.setvariable variable=AzDOBuildAccount;isOutput=true]$AzureDevOpsAccount" } catch { Write-Host $_ diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index e2ef5d192e4..9c566e8b8af 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -138,7 +138,13 @@ ChecksumsFeedOverride="$(ChecksumsFeedOverride)" TransportFeedOverride="$(TransportFeedOverride)" ShippingFeedOverride="$(ShippingFeedOverride)" - SymbolsFeedOverride="$(SymbolsFeedOverride)" /> + SymbolsFeedOverride="$(SymbolsFeedOverride)" + AzdoApiToken="$(AzdoApiToken)" + TemporaryStagingDir="$(TemporaryStagingDir)" + BuildId="$(BuildId)" + AzureDevOpsOrg="$(AzureDevOpsOrg)" + AzureProject="$(AzureProject)"/> + diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/BuildArtifacts.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/BuildArtifacts.cs new file mode 100644 index 00000000000..16df412ab49 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/BuildArtifacts.cs @@ -0,0 +1,35 @@ +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public class Properties + { + public string artifactsize { get; set; } + public string RootId { get; set; } + public string localpath { get; set; } + } + + public class Resource + { + public string type { get; set; } + public string data { get; set; } + public Properties properties { get; set; } + public string url { get; set; } + public string downloadUrl { get; set; } + } + + public class Value + { + public int id { get; set; } + public string name { get; set; } + public string source { get; set; } + public Resource resource { get; set; } + } + + public class BuildArtifacts + { + public int count { get; set; } + public IList value { get; set; } + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs index fef9005240e..ecc2a47dd0f 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs @@ -183,7 +183,17 @@ public string BuildQuality get { return _buildQuality.GetDescription(); } set { Enum.TryParse(value, true, out _buildQuality); } } + public string AzdoApiToken {get; set;} + public string StagingDir { get; set;} + + public string BuildId { get; set; } + + public string AzureProject { get; set; } + + public string AzureDevOpsOrg { get; set; } + + public bool UseApi {get; set;} /// /// Just an internal flag to keep track whether we published assets via a V3 manifest or not. /// @@ -372,7 +382,14 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui ChecksumsFeedOverride = this.ChecksumsFeedOverride, ShippingFeedOverride = this.ShippingFeedOverride, TransportFeedOverride = this.TransportFeedOverride, - SymbolsFeedOverride = this.SymbolsFeedOverride + SymbolsFeedOverride = this.SymbolsFeedOverride, + StagingDir = this.StagingDir, + AzdoApiToken = this.AzdoApiToken, + BuildId = this.BuildId, + AzureProject = this.AzureProject, + AzureDevOpsOrg = this.AzureDevOpsOrg, + UseApi = this.UseApi + }; } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 722e9e011f7..b7222b496f1 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -7,6 +7,7 @@ using System.Collections.Immutable; using System.IO; using System.Linq; +using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Reflection; @@ -14,11 +15,14 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using System.Xml; using Microsoft.Build.Framework; using Microsoft.DotNet.Build.Tasks.Feed.Model; +using Microsoft.DotNet.Build.Tasks.Feed.src; using Microsoft.DotNet.Maestro.Client; using Microsoft.DotNet.Maestro.Client.Models; using Microsoft.DotNet.VersionTools.BuildManifest.Model; +using Newtonsoft.Json; using NuGet.Packaging.Core; using NuGet.Versioning; using static Microsoft.DotNet.Build.Tasks.Feed.GeneralUtils; @@ -103,6 +107,22 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities public string BuildQuality { get; set; } + public string AzdoApiToken { get; set; } + + public string StagingDir { get; set; } + + public string AzureDevOpsFeedsApiVersion { get; set; } = "6.0"; + + public string AzureApiVersionForFileDownload { get; set; } = "4.1-preview.4"; + + public string AzureProject { get; set; } + + public string BuildId { get; set; } + + public string AzureDevOpsOrg { get; set; } + + private readonly string AzureDevOpsBaseUrl = $"https://dev.azure.com"; + public readonly Dictionary> FeedConfigs = new Dictionary>(); @@ -316,6 +336,133 @@ public void CheckForStableAssetsInNonIsolatedFeeds() } } + public HashSet ParseXmlFile(string pathToXml) + { + XmlDocument xmlDoc = new XmlDocument(); + xmlDoc.Load(pathToXml); + XmlNodeList itemsToSign = xmlDoc.GetElementsByTagName("ItemsToSign"); + XmlNodeList package = xmlDoc.GetElementsByTagName("Package"); + XmlNodeList blob = xmlDoc.GetElementsByTagName("Blob"); + //HashSet packageFiles = new HashSet(); + HashSet blobFiles = new HashSet(); + + /*for (int i = 0; i < package.Count; i++) + { + string fileName = ""; + fileName = $"{package[i].Attributes["Id"].Value}.{package[i].Attributes["Version"].Value}.nupkg"; + packageFiles.Add(fileName); + }*/ + + for (int i = 0; i < blob.Count; i++) + { + var blobLocation = blob[i].Attributes["Id"].Value.ToString(); + var segments = blobLocation.Split('/'); + var fileName = segments[segments.Length - 1]; + if (fileName.Contains(".symbols.nupkg")) + { + blobFiles.Add(fileName); + } + } + return blobFiles; + } + + public async Task HandleSymbolPublishingOneByOneAsync( + string pdbArtifactsBasePath, + string msdlToken, + string symWebToken, + string symbolPublishingExclusionsFile, + bool publishSpecialClrFiles) + { + + StringBuilder symbolLog = new StringBuilder(); + symbolLog.AppendLine("Publishing Symbols to Symbol server: "); + + var category = TargetFeedContentType.Symbols; + + string containerId = GetContainerId().Result; + string temporarySymbDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", "tempSymb")); + EnsureTemporaryDirectoryExists(temporarySymbDirectory); + HashSet blobs = new HashSet(); + using (HttpClientHandler handler = new HttpClientHandler {CheckCertificateRevocationList = true}) + { + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) + { + + string path = await DownloadFileAsync(client, "BlobArtifacts", containerId, "MergedManifest.xml", + temporarySymbDirectory); + blobs = ParseXmlFile(path); + DeleteTemporaryFiles(temporarySymbDirectory); + + Log.LogMessage(MessageImportance.High, $"Total number of symbol files : {blobs.Count}"); + } + } + + HashSet feedConfigsForSymbols = FeedConfigs[category]; + + Dictionary serversToPublish = + GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); + + IEnumerable filesToSymbolServer = null; + if (Directory.Exists(pdbArtifactsBasePath)) + { + var pdbEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.pdb", + System.IO.SearchOption.AllDirectories); + var dllEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.dll", + System.IO.SearchOption.AllDirectories); + filesToSymbolServer = pdbEntries.Concat(dllEntries); + } + + if (Directory.Exists(temporarySymbDirectory) && !string.IsNullOrEmpty(containerId)) + { + using HttpClientHandler handler = new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + CheckCertificateRevocationList = true + }; + string localBlobPath = ""; + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + foreach (var blob in blobs) + { + localBlobPath = await DownloadFileAsync(client,"BlobArtifacts", containerId, blob, + temporarySymbDirectory); + IEnumerable symbolFile = new List(); + symbolFile.ToList().Add(localBlobPath); + foreach (var server in serversToPublish) + { + var serverPath = server.Key; + var token = server.Value; + symbolLog.AppendLine($"Publishing symbol file {blob} to {serverPath}:"); + + await PublishSymbolsHelper.PublishAsync( + Log, + serverPath, + token, + symbolFile, + filesToSymbolServer, + null, + ExpirationInDays, + false, + publishSpecialClrFiles, + null, + false, + false, + true); + } + DeleteTemporaryFile(localBlobPath); + } + + symbolLog.AppendLine( + $"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false "); + symbolLog.Append($"\nTotal number of symbol files : {blobs.Count}"); + symbolLog.AppendLine("\nSuccessfully published to Symbol Server."); + symbolLog.AppendLine(); + Log.LogMessage(MessageImportance.High, symbolLog.ToString()); + symbolLog.Clear(); + DeleteTemporaryDirectory(Path.GetFullPath(Path.Combine(StagingDir, @"..\", "tempSymb"))); + } + } + /// /// Publishes symbol, dll and pdb files to symbol server. /// @@ -331,13 +478,13 @@ public async Task HandleSymbolPublishingAsync ( string msdlToken, string symWebToken, string symbolPublishingExclusionsFile, - string temporarySymbolsLocation, - bool publishSpecialClrFiles) + bool publishSpecialClrFiles, + string temporarySymbolsLocation = null) { StringBuilder symbolLog = new StringBuilder(); symbolLog.AppendLine("Publishing Symbols to Symbol server: "); - if (Directory.Exists(temporarySymbolsLocation)) + if (Directory.Exists(temporarySymbolsLocation) && Directory.EnumerateFileSystemEntries(PackageAssetsBasePath).Any()) { string[] fileEntries = Directory.GetFiles(temporarySymbolsLocation); @@ -385,7 +532,8 @@ await PublishSymbolsHelper.PublishAsync( } else { - Log.LogError($"Temporary symbols directory {temporarySymbolsLocation} does not exists."); + await HandleSymbolPublishingOneByOneAsync(pdbArtifactsBasePath, msdlToken, + symWebToken, symbolPublishingExclusionsFile, publishSpecialClrFiles); } } @@ -442,17 +590,42 @@ protected async Task HandlePackagePublishingAsync(Dictionary FilterPackages(HashSet packages, TargetFeedConfig feedConfig) @@ -479,6 +654,88 @@ private HashSet FilterPackages(HashSet + /// Gets the container Id, that is going to be used in another API call to download the assets + /// + /// + public async Task GetContainerId() + { + using (HttpClientHandler handler = new HttpClientHandler { CheckCertificateRevocationList = true }) + { + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) + { + string uri = + $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/{AzureProject}/_apis/build/builds/{BuildId}/artifacts?api-version={AzureDevOpsFeedsApiVersion}"; + HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); + HttpResponseMessage response = await client.SendAsync(getMessage); + response.EnsureSuccessStatusCode(); + string responseBody = await response.Content.ReadAsStringAsync(); + BuildArtifacts buildArtifacts = JsonConvert.DeserializeObject(responseBody); + string containerId = ""; + foreach (var bd in buildArtifacts.value) + { + if (string.Equals(bd.name, "BlobArtifacts")) + { + string[] segment = bd.resource.data.Split('/'); + containerId = segment[1]; + break; + } + } + return containerId; + } + } + } + + public async Task DownloadFileAsync(HttpClient client, string artifact, string containerId, + string fileName, string tempDirectory) + { + string uri = + $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/_apis/resources/Containers/{containerId}?itemPath=/{artifact}/{fileName}&isShallow=true&api-version={AzureApiVersionForFileDownload}"; + Log.LogMessage(MessageImportance.High, $"download file uri = {uri}"); + HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); + HttpResponseMessage response = await client.SendAsync(getMessage); + response.EnsureSuccessStatusCode(); + + string localPackagePath = Path.Combine(tempDirectory, fileName); + Log.LogMessage($"LocalPath to downloaded file {localPackagePath}"); + using (var fs = new FileStream(localPackagePath, FileMode.Create, + FileAccess.Write, FileShare.ReadWrite)) + { + using (var stream = await response.Content.ReadAsStreamAsync()) + { + { + await stream.CopyToAsync(fs); + fs.Flush(); + fs.Close(); + } + } + } + return localPackagePath; + } + protected async Task HandleBlobPublishingAsync(Dictionary> buildAssets) { List publishTasks = new List(); @@ -505,17 +762,44 @@ protected async Task HandleBlobPublishingAsync(Dictionary Log.LogMessage(MessageImportance.High, $"Blob {blob.Id} ({shippingString}) should go to {feedConfig.TargetURL} ({isolatedString}{internalString})"); } - switch (feedConfig.Type) + if (Directory.EnumerateFileSystemEntries(PackageAssetsBasePath).Any()) { - case FeedType.AzDoNugetFeed: - publishTasks.Add(PublishBlobsToAzDoNugetFeedAsync(filteredBlobs, buildAssets, feedConfig)); - break; - case FeedType.AzureStorageFeed: - publishTasks.Add(PublishBlobsToAzureStorageNugetFeedAsync(filteredBlobs, buildAssets, feedConfig)); - break; - default: - Log.LogError($"Unknown target feed type for category '{category}': '{feedConfig.Type}'."); - break; + switch (feedConfig.Type) + { + case FeedType.AzDoNugetFeed: + publishTasks.Add(PublishBlobsToAzDoNugetFeedAsync(filteredBlobs, buildAssets, + feedConfig)); + break; + case FeedType.AzureStorageFeed: + publishTasks.Add( + PublishBlobsToAzureStorageNugetFeedAsync(filteredBlobs, buildAssets, + feedConfig)); + break; + default: + Log.LogError( + $"Unknown target feed type for category '{category}': '{feedConfig.Type}'."); + break; + } + } + else + { + switch (feedConfig.Type) + { + case FeedType.AzDoNugetFeed: + publishTasks.Add(PublishBlobsToAzDoNugetFeedOneByOneAsync(filteredBlobs, buildAssets, + feedConfig)); + break; + case FeedType.AzureStorageFeed: + publishTasks.Add( + PublishBlobsToAzureStorageNugetFeedOneByOneAsync(filteredBlobs, buildAssets, + feedConfig)); + break; + default: + Log.LogError( + $"Unknown target feed type for category '{category}': '{feedConfig.Type}'."); + break; + } + } } } @@ -635,6 +919,27 @@ await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClien }); } + private async Task PublishPackagesToAzDoNugetFeedAsyncOneByOne( + HashSet packagesToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, + async (feed, httpClient, package, feedAccount, feedVisibility, feedName) => + { + string localPackagePath = Path.Combine(PackageAssetsBasePath, $"{package.Id}.{package.Version}.nupkg"); + if (!File.Exists(localPackagePath)) + { + Log.LogError($"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; + } + + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); + + await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, package.Version, feedAccount, feedVisibility, feedName); + }); + } + /// /// Push nuget packages to the azure devops feed. /// @@ -850,6 +1155,139 @@ await PushNugetPackagesAsync(packagesToPublish, feedConfig, m }); } + private async Task PublishBlobsToAzDoNugetFeedOneByOneAsync( + HashSet blobsToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + HashSet packagesToPublish = new HashSet(); + + foreach (var blob in blobsToPublish) + { + // Applies to symbol packages and core-sdk's VS feed packages + if (blob.Id.EndsWith(GeneralUtils.PackageSuffix, StringComparison.OrdinalIgnoreCase)) + { + packagesToPublish.Add(blob); + } + else + { + Log.LogWarning($"AzDO feed publishing not available for blobs. Blob '{blob.Id}' was not published."); + } + } + string containerId = GetContainerId().Result; + string temporaryBlobDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", "tempBlob")); + EnsureTemporaryDirectoryExists(temporaryBlobDirectory); + + + var parsedUri = Regex.Match(feedConfig.TargetURL, PublishingConstants.AzDoNuGetFeedPattern); + if (!parsedUri.Success) + { + Log.LogError($"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); + return; + } + string feedAccount = parsedUri.Groups["account"].Value; + string feedVisibility = parsedUri.Groups["visibility"].Value; + string feedName = parsedUri.Groups["feed"].Value; + if (Directory.Exists(temporaryBlobDirectory)) + { + using(HttpClient client = new HttpClient()) + { + foreach (var blob in blobsToPublish) + { + if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container)) + { + string fileName = Path.GetFileName(blob.Id); + string localBlobPath = await DownloadFileAsync(client,"BlobArtifacts", containerId, fileName, + temporaryBlobDirectory); + + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + return; + } + string id; + string version; + using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) + { + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); + } + + using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) + { + using (HttpClient httpClient = new HttpClient(new HttpClientHandler + {CheckCertificateRevocationList = true})) + { + httpClient.Timeout = TimeSpan.FromSeconds(180); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); + try + { + // Wait to avoid starting too many processes. + await clientThrottle.WaitAsync(); + await PushNugetPackageAsync(feedConfig, httpClient, localBlobPath, id, version, feedAccount, feedVisibility, feedName); + } + finally + { + clientThrottle.Release(); + } + + } + } + } + + } + } + + } + } + + private async Task PublishPackagesToAzureStorageNugetFeedOneByOneAsync( + HashSet packagesToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + string containerId = GetContainerId().Result; + var blobFeedAction = CreateBlobFeedAction(feedConfig); + + var pushOptions = new PushOptions + { + AllowOverwrite = feedConfig.AllowOverwrite, + PassIfExistingItemIdentical = true + }; + string temporaryPackageDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", "tempPackage")); + EnsureTemporaryDirectoryExists(temporaryPackageDirectory); + if (Directory.Exists(temporaryPackageDirectory)) + { + using (HttpClient client = new HttpClient()) + { + foreach (var package in packagesToPublish) + { + var packageFilename = $"{package.Id}.{package.Version}.nupkg"; + string localPackagePath = + await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, + temporaryPackageDirectory); + + if (!File.Exists(localPackagePath)) + { + Log.LogError( + $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; + } + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); + IEnumerable item = new List(); + item.ToList().Add(localPackagePath); + await blobFeedAction.PushToFeedAsync(item, pushOptions); + } + } + } + } + private async Task PublishPackagesToAzureStorageNugetFeedAsync( HashSet packagesToPublish, Dictionary> buildAssets, @@ -885,6 +1323,54 @@ private async Task PublishPackagesToAzureStorageNugetFeedAsync( await blobFeedAction.PushToFeedAsync(packages, pushOptions); } + private async Task PublishBlobsToAzureStorageNugetFeedOneByOneAsync(HashSet blobsToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + string containerId = GetContainerId().Result; + var blobFeedAction = CreateBlobFeedAction(feedConfig); + var pushOptions = new PushOptions + { + AllowOverwrite = feedConfig.AllowOverwrite, + PassIfExistingItemIdentical = true + }; + + foreach (var blob in blobsToPublish) + + { + var fileName = Path.GetFileName(blob.Id); + var localBlobPath = Path.Combine(BlobAssetsBasePath, fileName); + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + } + + var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, new Dictionary + { + {"RelativeBlobPath", blob.Id} + }); + + + TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container); + + using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) + { + await blobFeedAction.UploadAssetAsync(item, clientThrottle, pushOptions); + } + DeleteTemporaryFile(localBlobPath); + } + + if (LinkManager == null) + { + LinkManager = new LatestLinksManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, AkaMSGroupOwner, AkaMSCreatedBy, AkaMsOwners, Log); + } + + // The latest links should be updated only after the publishing is complete, to avoid + // dead links in the interim. + await LinkManager.CreateOrUpdateLatestLinksAsync(blobsToPublish, feedConfig, PublishingConstants.ExpectedFeedUrlSuffix.Length); + } + private async Task PublishBlobsToAzureStorageNugetFeedAsync( HashSet blobsToPublish, Dictionary> buildAssets, @@ -973,6 +1459,143 @@ private BlobFeedAction CreateBlobFeedAction(TargetFeedConfig feedConfig) return null; } } + private async Task PublishPackagesToAzDoNugetFeedOneByOneAsync(HashSet packagesToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + + var parsedUri = Regex.Match(feedConfig.TargetURL, PublishingConstants.AzDoNuGetFeedPattern); + if (!parsedUri.Success) + { + Log.LogError( + $"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); + return; + } + + string feedAccount = parsedUri.Groups["account"].Value; + string feedVisibility = parsedUri.Groups["visibility"].Value; + string feedName = parsedUri.Groups["feed"].Value; + + string temporaryPackageDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", "tempPackage")); + EnsureTemporaryDirectoryExists(temporaryPackageDirectory); + string containerId = GetContainerId().Result; + if (Directory.Exists(temporaryPackageDirectory)) + { + using HttpClientHandler handler = new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + CheckCertificateRevocationList = true + }; + string localPackagePath = ""; + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + { + foreach (var package in packagesToPublish) + { + var packageFilename = $"{package.Id}.{package.Version}.nupkg"; + + localPackagePath = await DownloadFileAsync(client, "PackageArtifacts", containerId, + packageFilename, + temporaryPackageDirectory); + + if (!File.Exists(localPackagePath)) + { + Log.LogError( + $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; + } + + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); + + using (HttpClient httpClient = new HttpClient(new HttpClientHandler + {CheckCertificateRevocationList = true})) + { + httpClient.Timeout = TimeSpan.FromSeconds(180); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String( + Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); + await PushNugetPackageAsync(feedConfig, httpClient, localPackagePath, package.Id, + package.Version, + feedAccount, feedVisibility, feedName); + } + DeleteTemporaryFile(packageFilename); + } + } + } + else + { + Log.LogError($"Temporary directory {temporaryPackageDirectory} does not exist"); + } + } + + /// + /// Create Temporary directory if it does not exists. + /// + /// + public void EnsureTemporaryDirectoryExists(string temporaryLocation) + { + if (!Directory.Exists(temporaryLocation)) + { + Directory.CreateDirectory(temporaryLocation); + } + } + + /// + /// Delete the files after publishing, this is part of cleanup + /// + /// + public void DeleteTemporaryFiles(string temporaryLocation) + { + try + { + if (Directory.Exists(temporaryLocation)) + { + string[] fileEntries = Directory.GetFiles(temporaryLocation); + foreach (var file in fileEntries) + { + File.Delete(file); + } + } + } + catch (Exception ex) + { + Log.LogWarning(ex.Message); + } + } + + /// + /// Delete the files after publishing, this is part of cleanup + /// + /// + public void DeleteTemporaryFile(string filePath) + { + try + { + if (File.Exists(filePath)) + { + Log.LogMessage($"Going to delete the following file {filePath}"); + File.Delete(filePath); + } + } + catch (Exception ex) + { + Log.LogWarning(ex.Message); + } + } + + /// + /// Deletes the temporary folder, this is part of clean up + /// + /// + public void DeleteTemporaryDirectory(string temporaryLocation) + { + if (Directory.Exists(temporaryLocation)) + { + Directory.Delete(temporaryLocation); + } + } protected bool AnyMissingRequiredProperty() { diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index 1aa5bb834ed..3dc1cdc65a6 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -63,6 +63,8 @@ public class PublishArtifactsInManifestV3 : PublishArtifactsInManifestBase public string SymbolsFeedOverride { get; set; } + public bool UseApi {get; set;} + public override bool Execute() { ExecuteAsync().GetAwaiter().GetResult(); @@ -98,17 +100,7 @@ public override async Task ExecuteAsync() return false; } - string temporarySymbolsLocation = - Path.GetFullPath(Path.Combine(BlobAssetsBasePath, @"..\", "tempSymbols")); - - EnsureTemporarySymbolDirectoryExists(temporarySymbolsLocation); - SplitArtifactsInCategories(BuildModel); - DeleteSymbolTemporaryFiles(temporarySymbolsLocation); - - //Copying symbol files to temporary location is required because the symUploader API needs read/write access to the files, - //since we publish blobs and symbols in parallel this will cause IO exceptions. - CopySymbolFilesToTemporaryLocation(BuildModel, temporarySymbolsLocation); if (Log.HasLoggedErrors) { @@ -199,16 +191,30 @@ public override async Task ExecuteAsync() return false; } + string temporarySymbolsLocation =""; + if(Directory.EnumerateFileSystemEntries(PackageAssetsBasePath).Any()) + { + + temporarySymbolsLocation = Path.GetFullPath(Path.Combine(BlobAssetsBasePath, @"..\", "tempSymbols")); + + EnsureTemporaryDirectoryExists(temporarySymbolsLocation); + DeleteTemporaryFiles(temporarySymbolsLocation); + + //Copying symbol files to temporary location is required because the symUploader API needs read/write access to the files, + //since we publish blobs and symbols in parallel this will cause IO exceptions. + CopySymbolFilesToTemporaryLocation(BuildModel, temporarySymbolsLocation); + + await Task.WhenAll(new Task[] { HandlePackagePublishingAsync(buildAssets), HandleBlobPublishingAsync(buildAssets), HandleSymbolPublishingAsync(PdbArtifactsBasePath, MsdlToken, - SymWebToken, SymbolPublishingExclusionsFile, temporarySymbolsLocation, PublishSpecialClrFiles) + SymWebToken, SymbolPublishingExclusionsFile, PublishSpecialClrFiles,temporarySymbolsLocation) }); - DeleteSymbolTemporaryFiles(temporarySymbolsLocation); - DeleteSymbolTemporaryDirectory(temporarySymbolsLocation); - Log.LogMessage(MessageImportance.High, "Successfully deleted the temporary symbols directory."); + DeleteTemporaryFiles(temporarySymbolsLocation); + DeleteTemporaryDirectory(temporarySymbolsLocation); + } await PersistPendingAssetLocationAsync(client); } catch (Exception e) @@ -245,52 +251,6 @@ private void CopySymbolFilesToTemporaryLocation(BuildModel buildModel, string sy } } - /// - /// Create Temporary Symbols directory if it does not exists. - /// - /// - public void EnsureTemporarySymbolDirectoryExists(string temporarySymbolsLocation) - { - if (!Directory.Exists(temporarySymbolsLocation)) - { - Directory.CreateDirectory(temporarySymbolsLocation); - } - } - /// - /// Delete the symbols files after publishing to Symbol server(s), this is part of cleanup - /// - /// - public void DeleteSymbolTemporaryFiles(string temporarySymbolsLocation) - { - try - { - if (Directory.Exists(temporarySymbolsLocation)) - { - string[] fileEntries = Directory.GetFiles(temporarySymbolsLocation); - foreach (var file in fileEntries) - { - File.Delete(file); - } - } - } - catch (Exception ex) - { - Log.LogWarning(ex.Message); - } - } - - /// - /// Deletes the temporary symbol folder, this is part of clean up - /// - /// - public void DeleteSymbolTemporaryDirectory(string temporarySymbolLocation) - { - if (Directory.Exists(temporarySymbolLocation)) - { - Directory.Delete(temporarySymbolLocation); - } - } - public string GetFeed(string feed, string feedOverride) { return (AllowFeedOverrides && !string.IsNullOrEmpty(feedOverride)) ? feedOverride : feed; From 819783474318b34cb189f9f72b35d880b8c50ebd Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Wed, 10 Mar 2021 10:09:11 -0800 Subject: [PATCH 02/32] remove unwanted param --- .../src/PublishArtifactsInManifest.cs | 5 +---- .../src/PublishArtifactsInManifestV3.cs | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs index ecc2a47dd0f..2475307491e 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs @@ -193,7 +193,6 @@ public string BuildQuality public string AzureDevOpsOrg { get; set; } - public bool UseApi {get; set;} /// /// Just an internal flag to keep track whether we published assets via a V3 manifest or not. /// @@ -387,9 +386,7 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui AzdoApiToken = this.AzdoApiToken, BuildId = this.BuildId, AzureProject = this.AzureProject, - AzureDevOpsOrg = this.AzureDevOpsOrg, - UseApi = this.UseApi - + AzureDevOpsOrg = this.AzureDevOpsOrg }; } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index 3dc1cdc65a6..f8f4fa8e72b 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -63,8 +63,6 @@ public class PublishArtifactsInManifestV3 : PublishArtifactsInManifestBase public string SymbolsFeedOverride { get; set; } - public bool UseApi {get; set;} - public override bool Execute() { ExecuteAsync().GetAwaiter().GetResult(); From 9d3a66e65b286166ed69d5d8e7d1c5a7c736e294 Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Wed, 10 Mar 2021 14:17:15 -0800 Subject: [PATCH 03/32] some fixes --- .../PublishToSymbolServerTest.cs | 6 +- .../src/PublishArtifactsInManifestBase.cs | 310 +++++++++++------- 2 files changed, 186 insertions(+), 130 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs index 3216cf23dba..a7178917fa2 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs @@ -85,7 +85,7 @@ public void TemporarySymbolDirectoryDoesNotExists() BuildEngine = buildEngine, }; var path = TestInputs.GetFullPath("Symbol"); - var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", path, false); + var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, path); Assert.True(task.Log.HasLoggedErrors); } @@ -94,9 +94,9 @@ public void TemporarySymbolsDirectoryTest() { var publishTask = new PublishArtifactsInManifestV3(); var path = TestInputs.GetFullPath("Test"); - publishTask.EnsureTemporarySymbolDirectoryExists(path); + publishTask.EnsureTemporaryDirectoryExists(path); Assert.True(Directory.Exists(path)); - publishTask.DeleteSymbolTemporaryDirectory(path); + publishTask.DeleteTemporaryDirectory(path); Assert.False(Directory.Exists(path)); } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index b7222b496f1..5f5dd5d3996 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -146,6 +146,8 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities private const int ExpirationInDays = 3650; + public int TimeoutInSeconds { get; set; } = 300; + protected LatestLinksManager LinkManager { get; set; } = null; /// @@ -384,18 +386,17 @@ public async Task HandleSymbolPublishingOneByOneAsync( Path.GetFullPath(Path.Combine(StagingDir, @"..\", "tempSymb")); EnsureTemporaryDirectoryExists(temporarySymbDirectory); HashSet blobs = new HashSet(); - using (HttpClientHandler handler = new HttpClientHandler {CheckCertificateRevocationList = true}) + using HttpClientHandler _handler = new HttpClientHandler() { - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) - { - - string path = await DownloadFileAsync(client, "BlobArtifacts", containerId, "MergedManifest.xml", - temporarySymbDirectory); - blobs = ParseXmlFile(path); - DeleteTemporaryFiles(temporarySymbDirectory); - - Log.LogMessage(MessageImportance.High, $"Total number of symbol files : {blobs.Count}"); - } + CheckCertificateRevocationList = true + }; + using (HttpClient client = CreateHttpClient(_handler, AzureDevOpsOrg, AzureProject)) + { + string path = await DownloadFileAsync(client, "BlobArtifacts", containerId, "MergedManifest.xml", + temporarySymbDirectory); + blobs = ParseXmlFile(path); + DeleteTemporaryFile(path); + Log.LogMessage(MessageImportance.High, $"Total number of symbol files : {blobs.Count}"); } HashSet feedConfigsForSymbols = FeedConfigs[category]; @@ -422,36 +423,39 @@ public async Task HandleSymbolPublishingOneByOneAsync( }; string localBlobPath = ""; using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) - foreach (var blob in blobs) { - localBlobPath = await DownloadFileAsync(client,"BlobArtifacts", containerId, blob, - temporarySymbDirectory); - IEnumerable symbolFile = new List(); - symbolFile.ToList().Add(localBlobPath); - foreach (var server in serversToPublish) + + foreach (var blob in blobs) { - var serverPath = server.Key; - var token = server.Value; - symbolLog.AppendLine($"Publishing symbol file {blob} to {serverPath}:"); - - await PublishSymbolsHelper.PublishAsync( - Log, - serverPath, - token, - symbolFile, - filesToSymbolServer, - null, - ExpirationInDays, - false, - publishSpecialClrFiles, - null, - false, - false, - true); + localBlobPath = await DownloadFileAsync(client, "BlobArtifacts", containerId, blob, + temporarySymbDirectory); + IEnumerable symbolFile = new List(); + symbolFile.ToList().Add(localBlobPath); + foreach (var server in serversToPublish) + { + var serverPath = server.Key; + var token = server.Value; + symbolLog.AppendLine($"Publishing symbol file {blob} to {serverPath}:"); + + await PublishSymbolsHelper.PublishAsync( + Log, + serverPath, + token, + symbolFile, + filesToSymbolServer, + null, + ExpirationInDays, + false, + publishSpecialClrFiles, + null, + false, + false, + true); + } + + DeleteTemporaryFile(localBlobPath); } - DeleteTemporaryFile(localBlobPath); } - symbolLog.AppendLine( $"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false "); symbolLog.Append($"\nTotal number of symbol files : {blobs.Count}"); @@ -674,6 +678,7 @@ public HttpClient CreateHttpClient(HttpClientHandler handler, string accountName client.DefaultRequestHeaders.Add( "Accept", $"application/xml;api-version={versionOverride ?? AzureDevOpsFeedsApiVersion}"); + client.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); return client; } @@ -1178,66 +1183,44 @@ private async Task PublishBlobsToAzDoNugetFeedOneByOneAsync( string temporaryBlobDirectory = Path.GetFullPath(Path.Combine(StagingDir, @"..\", "tempBlob")); EnsureTemporaryDirectoryExists(temporaryBlobDirectory); - - - var parsedUri = Regex.Match(feedConfig.TargetURL, PublishingConstants.AzDoNuGetFeedPattern); - if (!parsedUri.Success) - { - Log.LogError($"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); - return; - } - string feedAccount = parsedUri.Groups["account"].Value; - string feedVisibility = parsedUri.Groups["visibility"].Value; - string feedName = parsedUri.Groups["feed"].Value; if (Directory.Exists(temporaryBlobDirectory)) { - using(HttpClient client = new HttpClient()) + using HttpClientHandler handler = new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + CheckCertificateRevocationList = true + }; + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) { + client.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); foreach (var blob in blobsToPublish) { if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.Container)) { string fileName = Path.GetFileName(blob.Id); - string localBlobPath = await DownloadFileAsync(client,"BlobArtifacts", containerId, fileName, - temporaryBlobDirectory); - - if (!File.Exists(localBlobPath)) - { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); - return; - } - string id; - string version; - using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) - { - PackageIdentity packageIdentity = packageReader.GetIdentity(); - id = packageIdentity.Id; - version = packageIdentity.Version.ToString(); - } + + + string localBlobPath = await DownloadFileAsync(client, "BlobArtifacts", containerId, + fileName, + temporaryBlobDirectory); - using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) - { - using (HttpClient httpClient = new HttpClient(new HttpClientHandler - {CheckCertificateRevocationList = true})) + if (!File.Exists(localBlobPath)) { - httpClient.Timeout = TimeSpan.FromSeconds(180); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( - "Basic", - Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); - try - { - // Wait to avoid starting too many processes. - await clientThrottle.WaitAsync(); - await PushNugetPackageAsync(feedConfig, httpClient, localBlobPath, id, version, feedAccount, feedVisibility, feedName); - } - finally - { - clientThrottle.Release(); - } + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + } + string id; + string version; + using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) + { + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); } - } + + await PushBlobToNugetFeed(feedConfig, localBlobPath, id, version); + DeleteTemporaryFile(localBlobPath); } } @@ -1246,6 +1229,44 @@ private async Task PublishBlobsToAzDoNugetFeedOneByOneAsync( } } + + private async Task PushBlobToNugetFeed(TargetFeedConfig feedConfig, string localBlobPath, string id , string version) + { + var parsedUri = Regex.Match(feedConfig.TargetURL, PublishingConstants.AzDoNuGetFeedPattern); + if (!parsedUri.Success) + { + Log.LogError($"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); + return; + } + string feedAccount = parsedUri.Groups["account"].Value; + string feedVisibility = parsedUri.Groups["visibility"].Value; + string feedName = parsedUri.Groups["feed"].Value; + using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) + { + using (HttpClient httpClient = new HttpClient(new HttpClientHandler + { CheckCertificateRevocationList = true })) + { + httpClient.Timeout = TimeSpan.FromSeconds(180); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String( + Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", + feedConfig.Token)))); + try + { + // Wait to avoid starting too many processes. + await clientThrottle.WaitAsync(); + await PushNugetPackageAsync(feedConfig, httpClient, localBlobPath, id, + version, feedAccount, feedVisibility, feedName); + } + finally + { + clientThrottle.Release(); + } + + } + } + } private async Task PublishPackagesToAzureStorageNugetFeedOneByOneAsync( HashSet packagesToPublish, Dictionary> buildAssets, @@ -1264,12 +1285,19 @@ private async Task PublishPackagesToAzureStorageNugetFeedOneByOneAsync( EnsureTemporaryDirectoryExists(temporaryPackageDirectory); if (Directory.Exists(temporaryPackageDirectory)) { - using (HttpClient client = new HttpClient()) + using HttpClientHandler handler = new HttpClientHandler() { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + CheckCertificateRevocationList = true + }; + string localPackagePath = ""; + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + { + client.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); foreach (var package in packagesToPublish) { var packageFilename = $"{package.Id}.{package.Version}.nupkg"; - string localPackagePath = + localPackagePath = await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, temporaryPackageDirectory); @@ -1283,6 +1311,7 @@ await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename IEnumerable item = new List(); item.ToList().Add(localPackagePath); await blobFeedAction.PushToFeedAsync(item, pushOptions); + DeleteTemporaryFile(localPackagePath); } } } @@ -1334,31 +1363,52 @@ private async Task PublishBlobsToAzureStorageNugetFeedOneByOneAsync(HashSet + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + CheckCertificateRevocationList = true + }; + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) { - {"RelativeBlobPath", blob.Id} - }); + client.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); + { + foreach (var blob in blobsToPublish) + { + var fileName = Path.GetFileName(blob.Id); + var localBlobPath = + await DownloadFileAsync(client, "BlobArtifacts", containerId, fileName, + temporaryBlobDirectory); + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + } - TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.Container); + var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, + new Dictionary + { + {"RelativeBlobPath", blob.Id} + }); - using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) - { - await blobFeedAction.UploadAssetAsync(item, clientThrottle, pushOptions); + + TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container); + + using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) + { + await blobFeedAction.UploadAssetAsync(item, clientThrottle, pushOptions); + } + + DeleteTemporaryFile(localBlobPath); + } + } } - DeleteTemporaryFile(localBlobPath); } if (LinkManager == null) @@ -1464,17 +1514,6 @@ private async Task PublishPackagesToAzDoNugetFeedOneByOneAsync(HashSet /// Create Temporary directory if it does not exists. /// From 271e5c2dd131d4713abb910abf999fa67f3c7b9c Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Wed, 10 Mar 2021 19:35:53 -0800 Subject: [PATCH 04/32] more update --- eng/publishing/v3/publish-assets.yml | 2 +- .../tools/SdkTasks/PublishArtifactsInManifest.proj | 2 +- .../src/PublishArtifactsInManifestBase.cs | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml index 798ce54fe70..3228e193954 100644 --- a/eng/publishing/v3/publish-assets.yml +++ b/eng/publishing/v3/publish-assets.yml @@ -106,7 +106,7 @@ jobs: /p:SymWebToken=$(symweb-symbol-server-pat) /p:BuildQuality='${{ parameters.buildQuality }}' /p:AzdoApiToken='$(dn-bot-dotnet-build-rw-code-rw)' - /p:TemporaryStagingDir='$(Build.ArtifactStagingDirectory)/' + /p:StagingDir='$(Build.ArtifactStagingDirectory)/' /p:BuildId='$(AzDOBuildId)' /p:AzureDevOpsOrg='dnceng' /p:AzureProject='$(AzDOProjectName)' diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index 9c566e8b8af..fe8bbbe934b 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -140,7 +140,7 @@ ShippingFeedOverride="$(ShippingFeedOverride)" SymbolsFeedOverride="$(SymbolsFeedOverride)" AzdoApiToken="$(AzdoApiToken)" - TemporaryStagingDir="$(TemporaryStagingDir)" + StagingDir="$(StagingDir)" BuildId="$(BuildId)" AzureDevOpsOrg="$(AzureDevOpsOrg)" AzureProject="$(AzureProject)"/> diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 5f5dd5d3996..f1bdbfc38ad 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -812,6 +812,8 @@ protected async Task HandleBlobPublishingAsync(Dictionary { Log.LogError($"No target feed configuration found for artifact category: '{category}'."); } + DeleteTemporaryFiles(Path.Combine(StagingDir, @"..\", "tempBlob")); + DeleteTemporaryDirectory(Path.Combine(StagingDir, @"..\", "tempBlob")); } await Task.WhenAll(publishTasks); From 23fd68d7aba15d7db6c54281f1af289b51163f8d Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 11 Mar 2021 00:22:38 -0800 Subject: [PATCH 05/32] minor fixes --- eng/publishing/v3/publish-assets.yml | 2 +- .../SdkTasks/PublishArtifactsInManifest.proj | 3 ++- .../src/PublishArtifactsInManifest.cs | 5 ++++- .../src/PublishArtifactsInManifestBase.cs | 22 ++++++------------- .../src/PublishArtifactsInManifestV3.cs | 9 ++++---- 5 files changed, 19 insertions(+), 22 deletions(-) diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml index 3228e193954..1fe3f1ae7fe 100644 --- a/eng/publishing/v3/publish-assets.yml +++ b/eng/publishing/v3/publish-assets.yml @@ -110,7 +110,7 @@ jobs: /p:BuildId='$(AzDOBuildId)' /p:AzureDevOpsOrg='dnceng' /p:AzureProject='$(AzDOProjectName)' - + /p:UseApiOverride='true' - template: /eng/common/templates/steps/publish-logs.yml parameters: StageLabel: '${{ parameters.stageName }}' diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index fe8bbbe934b..7f94a16b257 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -143,7 +143,8 @@ StagingDir="$(StagingDir)" BuildId="$(BuildId)" AzureDevOpsOrg="$(AzureDevOpsOrg)" - AzureProject="$(AzureProject)"/> + AzureProject="$(AzureProject)" + UseApiOverride="$(UseApiOverride)"/> diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs index 2475307491e..99b8fc473e4 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs @@ -193,6 +193,8 @@ public string BuildQuality public string AzureDevOpsOrg { get; set; } + public bool UseApiOverride {get; set;} + /// /// Just an internal flag to keep track whether we published assets via a V3 manifest or not. /// @@ -386,7 +388,8 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui AzdoApiToken = this.AzdoApiToken, BuildId = this.BuildId, AzureProject = this.AzureProject, - AzureDevOpsOrg = this.AzureDevOpsOrg + AzureDevOpsOrg = this.AzureDevOpsOrg, + UseApiOverride = this.UseApiOverride }; } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index f1bdbfc38ad..ee47305b601 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -123,6 +123,8 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities private readonly string AzureDevOpsBaseUrl = $"https://dev.azure.com"; + public bool UseApiOverride { get; set; } + public readonly Dictionary> FeedConfigs = new Dictionary>(); @@ -146,8 +148,6 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities private const int ExpirationInDays = 3650; - public int TimeoutInSeconds { get; set; } = 300; - protected LatestLinksManager LinkManager { get; set; } = null; /// @@ -388,6 +388,7 @@ public async Task HandleSymbolPublishingOneByOneAsync( HashSet blobs = new HashSet(); using HttpClientHandler _handler = new HttpClientHandler() { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, CheckCertificateRevocationList = true }; using (HttpClient client = CreateHttpClient(_handler, AzureDevOpsOrg, AzureProject)) @@ -488,7 +489,7 @@ public async Task HandleSymbolPublishingAsync ( StringBuilder symbolLog = new StringBuilder(); symbolLog.AppendLine("Publishing Symbols to Symbol server: "); - if (Directory.Exists(temporarySymbolsLocation) && Directory.EnumerateFileSystemEntries(PackageAssetsBasePath).Any()) + if (!UseApiOverride) { string[] fileEntries = Directory.GetFiles(temporarySymbolsLocation); @@ -594,7 +595,7 @@ protected async Task HandlePackagePublishingAsync(Dictionary FilterPackages(HashSet packages, TargetFeedConfig feedConfig) @@ -678,7 +677,6 @@ public HttpClient CreateHttpClient(HttpClientHandler handler, string accountName client.DefaultRequestHeaders.Add( "Accept", $"application/xml;api-version={versionOverride ?? AzureDevOpsFeedsApiVersion}"); - client.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); return client; } @@ -719,7 +717,7 @@ public async Task DownloadFileAsync(HttpClient client, string artifact, { string uri = $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/_apis/resources/Containers/{containerId}?itemPath=/{artifact}/{fileName}&isShallow=true&api-version={AzureApiVersionForFileDownload}"; - Log.LogMessage(MessageImportance.High, $"download file uri = {uri}"); + Log.LogMessage($"download file uri = {uri}"); HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); HttpResponseMessage response = await client.SendAsync(getMessage); response.EnsureSuccessStatusCode(); @@ -767,7 +765,7 @@ protected async Task HandleBlobPublishingAsync(Dictionary Log.LogMessage(MessageImportance.High, $"Blob {blob.Id} ({shippingString}) should go to {feedConfig.TargetURL} ({isolatedString}{internalString})"); } - if (Directory.EnumerateFileSystemEntries(PackageAssetsBasePath).Any()) + if (!UseApiOverride) { switch (feedConfig.Type) { @@ -812,8 +810,6 @@ protected async Task HandleBlobPublishingAsync(Dictionary { Log.LogError($"No target feed configuration found for artifact category: '{category}'."); } - DeleteTemporaryFiles(Path.Combine(StagingDir, @"..\", "tempBlob")); - DeleteTemporaryDirectory(Path.Combine(StagingDir, @"..\", "tempBlob")); } await Task.WhenAll(publishTasks); @@ -1194,7 +1190,6 @@ private async Task PublishBlobsToAzDoNugetFeedOneByOneAsync( }; using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) { - client.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); foreach (var blob in blobsToPublish) { if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, @@ -1295,7 +1290,6 @@ private async Task PublishPackagesToAzureStorageNugetFeedOneByOneAsync( string localPackagePath = ""; using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) { - client.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); foreach (var package in packagesToPublish) { var packageFilename = $"{package.Id}.{package.Version}.nupkg"; @@ -1378,7 +1372,6 @@ private async Task PublishBlobsToAzureStorageNugetFeedOneByOneAsync(HashSet ExecuteAsync() } string temporarySymbolsLocation =""; - if(Directory.EnumerateFileSystemEntries(PackageAssetsBasePath).Any()) + if (Directory.EnumerateFileSystemEntries(PackageAssetsBasePath).Any()) { - temporarySymbolsLocation = Path.GetFullPath(Path.Combine(BlobAssetsBasePath, @"..\", "tempSymbols")); + temporarySymbolsLocation = + Path.GetFullPath(Path.Combine(BlobAssetsBasePath, @"..\", "tempSymbols")); EnsureTemporaryDirectoryExists(temporarySymbolsLocation); DeleteTemporaryFiles(temporarySymbolsLocation); @@ -201,7 +202,7 @@ public override async Task ExecuteAsync() //Copying symbol files to temporary location is required because the symUploader API needs read/write access to the files, //since we publish blobs and symbols in parallel this will cause IO exceptions. CopySymbolFilesToTemporaryLocation(BuildModel, temporarySymbolsLocation); - + } await Task.WhenAll(new Task[] { HandlePackagePublishingAsync(buildAssets), @@ -212,7 +213,7 @@ await Task.WhenAll(new Task[] { DeleteTemporaryFiles(temporarySymbolsLocation); DeleteTemporaryDirectory(temporarySymbolsLocation); - } + await PersistPendingAssetLocationAsync(client); } catch (Exception e) From b82c1c0d213b8829aa7e9b7f80bccaca28f2790b Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 11 Mar 2021 13:25:55 -0800 Subject: [PATCH 06/32] Disabled package download --- eng/publishing/v3/publish-assets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml index 0912b5f8aa1..3d78cacb56d 100644 --- a/eng/publishing/v3/publish-assets.yml +++ b/eng/publishing/v3/publish-assets.yml @@ -30,9 +30,9 @@ jobs: - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true + enabled: false inputs: buildType: specific - enabled: false buildVersionToDownload: specific project: $(AzDOProjectName) pipeline: $(AzDOPipelineId) From ff093e14913e3f6f2c16c481c02a62dc3b31e2f3 Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 11 Mar 2021 14:42:48 -0800 Subject: [PATCH 07/32] fix --- .../src/PublishArtifactsInManifestBase.cs | 5 ++--- .../src/PublishArtifactsInManifestV3.cs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index ee47305b601..4a7139d66b3 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -486,11 +486,10 @@ public async Task HandleSymbolPublishingAsync ( bool publishSpecialClrFiles, string temporarySymbolsLocation = null) { - StringBuilder symbolLog = new StringBuilder(); - symbolLog.AppendLine("Publishing Symbols to Symbol server: "); - if (!UseApiOverride) { + StringBuilder symbolLog = new StringBuilder(); + symbolLog.AppendLine("Publishing Symbols to Symbol server: "); string[] fileEntries = Directory.GetFiles(temporarySymbolsLocation); var category = TargetFeedContentType.Symbols; diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index d6e83195e46..4dd62d1dff8 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -190,7 +190,7 @@ public override async Task ExecuteAsync() } string temporarySymbolsLocation =""; - if (Directory.EnumerateFileSystemEntries(PackageAssetsBasePath).Any()) + if (!UseApiOverride) { temporarySymbolsLocation = From 91f474a3a7105acca139b68a1098a1de36d4067c Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 11 Mar 2021 16:43:49 -0800 Subject: [PATCH 08/32] Remove some logging --- .../src/PublishArtifactsInManifestBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 4a7139d66b3..eaaee0a0918 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -1094,7 +1094,7 @@ public async Task PushNugetPackageAsync( } else { - Log.LogMessage(MessageImportance.High, $"Succeeded publishing package '{localPackageLocation}' to feed {feedConfig.TargetURL}"); + Log.LogMessage($"Succeeded publishing package '{localPackageLocation}' to feed {feedConfig.TargetURL}"); } } catch (Exception e) From 68e8ea86a6771a91590b0d064eea1fae1b9a7f67 Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Mon, 15 Mar 2021 23:03:13 -0700 Subject: [PATCH 09/32] More fix --- .../src/PublishArtifactsInManifestBase.cs | 557 +++++++++--------- 1 file changed, 283 insertions(+), 274 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index eaaee0a0918..c631f8025c6 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -125,6 +125,8 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities public bool UseApiOverride { get; set; } + public int Retries = 5; + public readonly Dictionary> FeedConfigs = new Dictionary>(); @@ -148,6 +150,8 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities private const int ExpirationInDays = 3650; + public int TimeoutInMinutes { get; set; } = 5; + protected LatestLinksManager LinkManager { get; set; } = null; /// @@ -337,38 +341,7 @@ public void CheckForStableAssetsInNonIsolatedFeeds() } } } - - public HashSet ParseXmlFile(string pathToXml) - { - XmlDocument xmlDoc = new XmlDocument(); - xmlDoc.Load(pathToXml); - XmlNodeList itemsToSign = xmlDoc.GetElementsByTagName("ItemsToSign"); - XmlNodeList package = xmlDoc.GetElementsByTagName("Package"); - XmlNodeList blob = xmlDoc.GetElementsByTagName("Blob"); - //HashSet packageFiles = new HashSet(); - HashSet blobFiles = new HashSet(); - - /*for (int i = 0; i < package.Count; i++) - { - string fileName = ""; - fileName = $"{package[i].Attributes["Id"].Value}.{package[i].Attributes["Version"].Value}.nupkg"; - packageFiles.Add(fileName); - }*/ - - for (int i = 0; i < blob.Count; i++) - { - var blobLocation = blob[i].Attributes["Id"].Value.ToString(); - var segments = blobLocation.Split('/'); - var fileName = segments[segments.Length - 1]; - if (fileName.Contains(".symbols.nupkg")) - { - blobFiles.Add(fileName); - } - } - return blobFiles; - } - - public async Task HandleSymbolPublishingOneByOneAsync( + public async Task HandleSymbolPublishingOneByOneAsync( string pdbArtifactsBasePath, string msdlToken, string symWebToken, @@ -379,92 +352,122 @@ public async Task HandleSymbolPublishingOneByOneAsync( StringBuilder symbolLog = new StringBuilder(); symbolLog.AppendLine("Publishing Symbols to Symbol server: "); - var category = TargetFeedContentType.Symbols; + var symbolCategory = TargetFeedContentType.Symbols; string containerId = GetContainerId().Result; - string temporarySymbDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", "tempSymb")); - EnsureTemporaryDirectoryExists(temporarySymbDirectory); - HashSet blobs = new HashSet(); - using HttpClientHandler _handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; - using (HttpClient client = CreateHttpClient(_handler, AzureDevOpsOrg, AzureProject)) + + HashSet symbolsToPublish = new HashSet(); + //Get all the symbol file names + foreach (var blobsPerCategory in BlobsByCategory) { - string path = await DownloadFileAsync(client, "BlobArtifacts", containerId, "MergedManifest.xml", - temporarySymbDirectory); - blobs = ParseXmlFile(path); - DeleteTemporaryFile(path); - Log.LogMessage(MessageImportance.High, $"Total number of symbol files : {blobs.Count}"); + var blobs = blobsPerCategory.Value; + foreach (var blob in blobs) + { + var segments = blob.ToString().Split('/'); + var fileName = segments[segments.Length - 1]; + if (fileName.Contains(".symbols.nupkg")) + { + symbolsToPublish.Add(fileName); + } + } } - HashSet feedConfigsForSymbols = FeedConfigs[category]; + HashSet feedConfigsForSymbols = FeedConfigs[symbolCategory]; Dictionary serversToPublish = GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); IEnumerable filesToSymbolServer = null; - if (Directory.Exists(pdbArtifactsBasePath)) - { - var pdbEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.pdb", - System.IO.SearchOption.AllDirectories); - var dllEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.dll", - System.IO.SearchOption.AllDirectories); - filesToSymbolServer = pdbEntries.Concat(dllEntries); - } - - if (Directory.Exists(temporarySymbDirectory) && !string.IsNullOrEmpty(containerId)) - { - using HttpClientHandler handler = new HttpClientHandler() + using HttpClientHandler handler = new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, CheckCertificateRevocationList = true }; - string localBlobPath = ""; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + { + string localSymbolPath = ""; + foreach (var symbol in symbolsToPublish) { - - foreach (var blob in blobs) + string temporarySymbDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporarySymbDirectory); + localSymbolPath = Path.Combine(temporarySymbDirectory, symbol); + await DownloadFileAsync(client, "BlobArtifacts", containerId, symbol, localSymbolPath); + + Log.LogMessage($"Local Symbol path to downloaded file {localSymbolPath}"); + IEnumerable symbolFile = new List(); + symbolFile.ToList().Add(localSymbolPath); + foreach (var server in serversToPublish) { - localBlobPath = await DownloadFileAsync(client, "BlobArtifacts", containerId, blob, - temporarySymbDirectory); - IEnumerable symbolFile = new List(); - symbolFile.ToList().Add(localBlobPath); - foreach (var server in serversToPublish) - { - var serverPath = server.Key; - var token = server.Value; - symbolLog.AppendLine($"Publishing symbol file {blob} to {serverPath}:"); - - await PublishSymbolsHelper.PublishAsync( - Log, - serverPath, - token, - symbolFile, - filesToSymbolServer, - null, - ExpirationInDays, - false, - publishSpecialClrFiles, - null, - false, - false, - true); - } - - DeleteTemporaryFile(localBlobPath); + var serverPath = server.Key; + var token = server.Value; + symbolLog.AppendLine($"Publishing symbol file {symbol} to {serverPath}:"); + + await PublishSymbolsHelper.PublishAsync( + Log, + serverPath, + token, + symbolFile, + filesToSymbolServer, + null, + ExpirationInDays, + false, + publishSpecialClrFiles, + null, + false, + false, + true); } + + DeleteTemporaryFile(localSymbolPath); + DeleteTemporaryDirectory(temporarySymbDirectory); } + symbolLog.AppendLine( $"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false "); - symbolLog.Append($"\nTotal number of symbol files : {blobs.Count}"); + symbolLog.Append($"\nTotal number of symbol files : {symbolsToPublish.Count}"); symbolLog.AppendLine("\nSuccessfully published to Symbol Server."); symbolLog.AppendLine(); Log.LogMessage(MessageImportance.High, symbolLog.ToString()); symbolLog.Clear(); - DeleteTemporaryDirectory(Path.GetFullPath(Path.Combine(StagingDir, @"..\", "tempSymb"))); + + } + + // publishing pdb artifacts + if (Directory.Exists(pdbArtifactsBasePath)) + { + var pdbEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.pdb", + System.IO.SearchOption.AllDirectories); + var dllEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.dll", + System.IO.SearchOption.AllDirectories); + filesToSymbolServer = pdbEntries.Concat(dllEntries); + } + + if (filesToSymbolServer != null && filesToSymbolServer.Any()) + { + IEnumerable symbolFile = null; + foreach (var server in serversToPublish) + { + var serverPath = server.Key; + var token = server.Value; + symbolLog.AppendLine($"Publishing pdbFiles to {serverPath}:"); + + await PublishSymbolsHelper.PublishAsync( + Log, + serverPath, + token, + symbolFile, + filesToSymbolServer, + null, + ExpirationInDays, + false, + publishSpecialClrFiles, + null, + false, + false, + true); + } + Log.LogMessage($"Successfully published pdb files"); } } @@ -691,8 +694,8 @@ public async Task GetContainerId() { string uri = $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/{AzureProject}/_apis/build/builds/{BuildId}/artifacts?api-version={AzureDevOpsFeedsApiVersion}"; - HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); - HttpResponseMessage response = await client.SendAsync(getMessage); + using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); + using HttpResponseMessage response = await client.SendAsync(getMessage); response.EnsureSuccessStatusCode(); string responseBody = await response.Content.ReadAsStringAsync(); BuildArtifacts buildArtifacts = JsonConvert.DeserializeObject(responseBody); @@ -711,32 +714,65 @@ public async Task GetContainerId() } } - public async Task DownloadFileAsync(HttpClient client, string artifact, string containerId, - string fileName, string tempDirectory) + public async Task DownloadFileAsync(HttpClient client, string artifact, string containerId, + string fileName, string path) { + int attempt = 0; string uri = $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/_apis/resources/Containers/{containerId}?itemPath=/{artifact}/{fileName}&isShallow=true&api-version={AzureApiVersionForFileDownload}"; Log.LogMessage($"download file uri = {uri}"); - HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); - HttpResponseMessage response = await client.SendAsync(getMessage); - response.EnsureSuccessStatusCode(); - - string localPackagePath = Path.Combine(tempDirectory, fileName); - Log.LogMessage($"LocalPath to downloaded file {localPackagePath}"); - using (var fs = new FileStream(localPackagePath, FileMode.Create, - FileAccess.Write, FileShare.ReadWrite)) + while (true) { - using (var stream = await response.Content.ReadAsStreamAsync()) + try + { + CancellationTokenSource timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes)); + using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); + using HttpResponseMessage response = await client.SendAsync(getMessage, timeoutTokenSource.Token); + if (response.StatusCode == HttpStatusCode.NotFound || + response.ReasonPhrase.StartsWith("The requested URI does not represent any resource on the server.", StringComparison.OrdinalIgnoreCase)) + { + Log.LogMessage($"Problems downloading file from '{uri}'. Does the resource exist on the storage? {response.StatusCode} : {response.ReasonPhrase}"); + return false; + } + + response.EnsureSuccessStatusCode(); + + using (var fs = new FileStream(path, FileMode.Create, + FileAccess.ReadWrite, FileShare.ReadWrite)) + { + using (var stream = await response.Content.ReadAsStreamAsync()) + { + try + { + await stream.CopyToAsync(fs); + } + finally + { + stream.Close(); + fs.Close(); + } + } + } + + return true; + } + + catch (Exception e) when (e is HttpRequestException || e is IOException && !(e is DirectoryNotFoundException || e is PathTooLongException)) + { + attempt++; + if(attempt > Retries) { - await stream.CopyToAsync(fs); - fs.Flush(); - fs.Close(); + Log.LogMessage($"Failed to download {uri} to {path} "); + return false; } + Log.LogMessage($"Retrying download of '{uri}' to '{path}' due to failure: '{e.Message}' ({attempt}/{Retries})"); + + await Task.Delay(RetryDelayMilliseconds).ConfigureAwait(false); + continue; } - } - return localPackagePath; } + } protected async Task HandleBlobPublishingAsync(Dictionary> buildAssets) { @@ -921,27 +957,6 @@ await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClien }); } - private async Task PublishPackagesToAzDoNugetFeedAsyncOneByOne( - HashSet packagesToPublish, - Dictionary> buildAssets, - TargetFeedConfig feedConfig) - { - await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, - async (feed, httpClient, package, feedAccount, feedVisibility, feedName) => - { - string localPackagePath = Path.Combine(PackageAssetsBasePath, $"{package.Id}.{package.Version}.nupkg"); - if (!File.Exists(localPackagePath)) - { - Log.LogError($"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); - return; - } - - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); - - await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, package.Version, feedAccount, feedVisibility, feedName); - }); - } - /// /// Push nuget packages to the azure devops feed. /// @@ -1156,7 +1171,6 @@ await PushNugetPackagesAsync(packagesToPublish, feedConfig, m } }); } - private async Task PublishBlobsToAzDoNugetFeedOneByOneAsync( HashSet blobsToPublish, Dictionary> buildAssets, @@ -1173,56 +1187,57 @@ private async Task PublishBlobsToAzDoNugetFeedOneByOneAsync( } else { - Log.LogWarning($"AzDO feed publishing not available for blobs. Blob '{blob.Id}' was not published."); + Log.LogWarning( + $"AzDO feed publishing not available for blobs. Blob '{blob.Id}' was not published."); } } + string containerId = GetContainerId().Result; - string temporaryBlobDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", "tempBlob")); - EnsureTemporaryDirectoryExists(temporaryBlobDirectory); - if (Directory.Exists(temporaryBlobDirectory)) + + + using HttpClientHandler handler = new HttpClientHandler() { - using HttpClientHandler handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + CheckCertificateRevocationList = true + }; + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) + { + foreach (var blob in blobsToPublish) { - foreach (var blob in blobsToPublish) + if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container)) { - if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.Container)) + string temporaryBlobDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryBlobDirectory); + string fileName = Path.GetFileName(blob.Id); + string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); + await DownloadFileAsync(client, "BlobArtifacts", containerId, + fileName, + localBlobPath); + + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + } + + string id; + string version; + using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) { - string fileName = Path.GetFileName(blob.Id); - - - string localBlobPath = await DownloadFileAsync(client, "BlobArtifacts", containerId, - fileName, - temporaryBlobDirectory); - - if (!File.Exists(localBlobPath)) - { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); - } - - string id; - string version; - using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) - { - PackageIdentity packageIdentity = packageReader.GetIdentity(); - id = packageIdentity.Id; - version = packageIdentity.Version.ToString(); - } - - await PushBlobToNugetFeed(feedConfig, localBlobPath, id, version); - DeleteTemporaryFile(localBlobPath); + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); } + await PushBlobToNugetFeed(feedConfig, localBlobPath, id, version); + DeleteTemporaryFile(localBlobPath); + DeleteTemporaryDirectory(temporaryBlobDirectory); } - } + } } + } @@ -1276,38 +1291,40 @@ private async Task PublishPackagesToAzureStorageNugetFeedOneByOneAsync( AllowOverwrite = feedConfig.AllowOverwrite, PassIfExistingItemIdentical = true }; - string temporaryPackageDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", "tempPackage")); - EnsureTemporaryDirectoryExists(temporaryPackageDirectory); - if (Directory.Exists(temporaryPackageDirectory)) + + + using HttpClientHandler handler = new HttpClientHandler() { - using HttpClientHandler handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; - string localPackagePath = ""; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + CheckCertificateRevocationList = true + }; + string localPackagePath = ""; + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + { + foreach (var package in packagesToPublish) { - foreach (var package in packagesToPublish) - { - var packageFilename = $"{package.Id}.{package.Version}.nupkg"; - localPackagePath = - await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, - temporaryPackageDirectory); + string temporaryPackageDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryPackageDirectory); + var packageFilename = $"{package.Id}.{package.Version}.nupkg"; + localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); + await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, + localPackagePath); - if (!File.Exists(localPackagePath)) - { - Log.LogError( - $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); - return; - } - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); - IEnumerable item = new List(); - item.ToList().Add(localPackagePath); - await blobFeedAction.PushToFeedAsync(item, pushOptions); - DeleteTemporaryFile(localPackagePath); + if (!File.Exists(localPackagePath)) + { + Log.LogError( + $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; } + + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); + IEnumerable item = new List(); + item.ToList().Add(localPackagePath); + await blobFeedAction.PushToFeedAsync(item, pushOptions); + DeleteTemporaryFile(localPackagePath); + DeleteTemporaryDirectory(temporaryPackageDirectory); } } } @@ -1348,8 +1365,8 @@ private async Task PublishPackagesToAzureStorageNugetFeedAsync( } private async Task PublishBlobsToAzureStorageNugetFeedOneByOneAsync(HashSet blobsToPublish, - Dictionary> buildAssets, - TargetFeedConfig feedConfig) + Dictionary> buildAssets, + TargetFeedConfig feedConfig) { string containerId = GetContainerId().Result; var blobFeedAction = CreateBlobFeedAction(feedConfig); @@ -1358,63 +1375,61 @@ private async Task PublishBlobsToAzureStorageNugetFeedOneByOneAsync(HashSet - { - {"RelativeBlobPath", blob.Id} - }); + var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, + new Dictionary + { + {"RelativeBlobPath", blob.Id} + }); - TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.Container); + TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container); - using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) - { - await blobFeedAction.UploadAssetAsync(item, clientThrottle, pushOptions); - } - - DeleteTemporaryFile(localBlobPath); + using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) + { + await blobFeedAction.UploadAssetAsync(item, clientThrottle, pushOptions); } + + DeleteTemporaryFile(localBlobPath); + DeleteTemporaryDirectory(temporaryBlobDirectory); } } } - if (LinkManager == null) { - LinkManager = new LatestLinksManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, AkaMSGroupOwner, AkaMSCreatedBy, AkaMsOwners, Log); + LinkManager = new LatestLinksManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, AkaMSGroupOwner, + AkaMSCreatedBy, AkaMsOwners, Log); } // The latest links should be updated only after the publishing is complete, to avoid // dead links in the interim. - await LinkManager.CreateOrUpdateLatestLinksAsync(blobsToPublish, feedConfig, PublishingConstants.ExpectedFeedUrlSuffix.Length); + await LinkManager.CreateOrUpdateLatestLinksAsync(blobsToPublish, feedConfig, + PublishingConstants.ExpectedFeedUrlSuffix.Length); } - private async Task PublishBlobsToAzureStorageNugetFeedAsync( HashSet blobsToPublish, Dictionary> buildAssets, @@ -1507,51 +1522,44 @@ private async Task PublishPackagesToAzDoNugetFeedOneByOneAsync(HashSet> buildAssets, TargetFeedConfig feedConfig) { - - string temporaryPackageDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", "tempPackage")); - EnsureTemporaryDirectoryExists(temporaryPackageDirectory); string containerId = GetContainerId().Result; - if (Directory.Exists(temporaryPackageDirectory)) + + using HttpClientHandler handler = new HttpClientHandler() { - using HttpClientHandler handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; - string localPackagePath = ""; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + CheckCertificateRevocationList = true + }; + string localPackagePath = ""; + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + { + foreach (var package in packagesToPublish) { - foreach (var package in packagesToPublish) - { - var packageFilename = $"{package.Id}.{package.Version}.nupkg"; - - localPackagePath = await DownloadFileAsync(client, "PackageArtifacts", containerId, - packageFilename, - temporaryPackageDirectory); + var packageFilename = $"{package.Id}.{package.Version}.nupkg"; + string temporaryPackageDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryPackageDirectory); + localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); + await DownloadFileAsync(client, "PackageArtifacts", containerId, + packageFilename, + localPackagePath); - if (!File.Exists(localPackagePath)) - { - Log.LogError( - $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); - return; - } - - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.NugetFeed); - await PushPackageToNugetFeed(feedConfig, localPackagePath, package.Id, package.Version); - DeleteTemporaryFile(packageFilename); + if (!File.Exists(localPackagePath)) + { + Log.LogError( + $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; } + + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); + await PushPackageToNugetFeed(feedConfig, localPackagePath, package.Id, package.Version); + File.Delete(localPackagePath); + Directory.Delete(temporaryPackageDirectory); } } - else - { - Log.LogError($"Temporary directory {temporaryPackageDirectory} does not exist"); - } } - private async Task PushPackageToNugetFeed(TargetFeedConfig feedConfig, string localPackagePath, string id , string version) { var parsedUri = Regex.Match(feedConfig.TargetURL, PublishingConstants.AzDoNuGetFeedPattern); @@ -1579,6 +1587,7 @@ await PushNugetPackageAsync(feedConfig, httpClient, localPackagePath, id, feedAccount, feedVisibility, feedName); } } + /// /// Create Temporary directory if it does not exists. /// From 037401b5efcb416675434fa9a1f4e7ee5493917a Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 18 Mar 2021 17:05:38 -0700 Subject: [PATCH 10/32] test sdk --- eng/Versions.props | 2 +- global.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index 0861ec398ad..fb5df08aa65 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -66,7 +66,7 @@ 2.4.1 2.0.3 2.4.1 - 6.0.0-beta.21166.5 + 6.0.0-beta.21168.4 6.0.0-beta.21166.5 1.22.0 1.1.2 diff --git a/global.json b/global.json index 18778689aad..ba2d76a6d88 100644 --- a/global.json +++ b/global.json @@ -3,7 +3,7 @@ "dotnet": "6.0.100-preview.2.21155.3" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21166.5", + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21168.4", "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21166.5" } } From 6d73f0bed552d1cd83d70eed0fa128b0dc8346b8 Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 18 Mar 2021 16:13:28 -0700 Subject: [PATCH 11/32] test --- .../PublishToSymbolServerTest.cs | 2 +- .../src/PublishArtifactsInManifestBase.cs | 561 ++++++++++-------- .../src/PublishArtifactsInManifestV3.cs | 2 +- .../src/{ => model}/BuildArtifacts.cs | 0 4 files changed, 325 insertions(+), 240 deletions(-) rename src/Microsoft.DotNet.Build.Tasks.Feed/src/{ => model}/BuildArtifacts.cs (100%) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs index a7178917fa2..a9bcf5f5cbc 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs @@ -85,7 +85,7 @@ public void TemporarySymbolDirectoryDoesNotExists() BuildEngine = buildEngine, }; var path = TestInputs.GetFullPath("Symbol"); - var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, path); + //var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, path); Assert.True(task.Log.HasLoggedErrors); } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index c631f8025c6..fdf41c3be4c 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -346,7 +346,8 @@ public async Task HandleSymbolPublishingOneByOneAsync( string msdlToken, string symWebToken, string symbolPublishingExclusionsFile, - bool publishSpecialClrFiles) + bool publishSpecialClrFiles, + Dictionary> buildAssets) { StringBuilder symbolLog = new StringBuilder(); @@ -354,21 +355,20 @@ public async Task HandleSymbolPublishingOneByOneAsync( var symbolCategory = TargetFeedContentType.Symbols; - string containerId = GetContainerId().Result; + string containerId = await GetContainerIdAsync(); HashSet symbolsToPublish = new HashSet(); //Get all the symbol file names - foreach (var blobsPerCategory in BlobsByCategory) + + foreach (var asset in buildAssets) { - var blobs = blobsPerCategory.Value; - foreach (var blob in blobs) + var name = asset.Key; + if (name.Contains(".symbols.nupkg")) { - var segments = blob.ToString().Split('/'); + var segments = name.Split('/'); var fileName = segments[segments.Length - 1]; - if (fileName.Contains(".symbols.nupkg")) - { - symbolsToPublish.Add(fileName); - } + symbolsToPublish.Add(fileName); + } } @@ -487,6 +487,7 @@ public async Task HandleSymbolPublishingAsync ( string symWebToken, string symbolPublishingExclusionsFile, bool publishSpecialClrFiles, + Dictionary> buildAssets, string temporarySymbolsLocation = null) { if (!UseApiOverride) @@ -540,7 +541,7 @@ await PublishSymbolsHelper.PublishAsync( else { await HandleSymbolPublishingOneByOneAsync(pdbArtifactsBasePath, msdlToken, - symWebToken, symbolPublishingExclusionsFile, publishSpecialClrFiles); + symWebToken, symbolPublishingExclusionsFile, publishSpecialClrFiles, buildAssets); } } @@ -594,46 +595,26 @@ protected async Task HandlePackagePublishingAsync(Dictionary /// - public async Task GetContainerId() + public async Task GetContainerIdAsync() { using (HttpClientHandler handler = new HttpClientHandler { CheckCertificateRevocationList = true }) { @@ -725,13 +706,17 @@ public async Task DownloadFileAsync(HttpClient client, string artifact, st { try { - CancellationTokenSource timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes)); + CancellationTokenSource timeoutTokenSource = + new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes)); using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); using HttpResponseMessage response = await client.SendAsync(getMessage, timeoutTokenSource.Token); if (response.StatusCode == HttpStatusCode.NotFound || - response.ReasonPhrase.StartsWith("The requested URI does not represent any resource on the server.", StringComparison.OrdinalIgnoreCase)) + response.ReasonPhrase.StartsWith( + "The requested URI does not represent any resource on the server.", + StringComparison.OrdinalIgnoreCase)) { - Log.LogMessage($"Problems downloading file from '{uri}'. Does the resource exist on the storage? {response.StatusCode} : {response.ReasonPhrase}"); + Log.LogMessage( + $"Problems downloading file from '{uri}'. Does the resource exist on the storage? {response.StatusCode} : {response.ReasonPhrase}"); return false; } @@ -757,22 +742,26 @@ public async Task DownloadFileAsync(HttpClient client, string artifact, st return true; } - catch (Exception e) when (e is HttpRequestException || e is IOException && !(e is DirectoryNotFoundException || e is PathTooLongException)) + catch (Exception e) when (e is HttpRequestException || + e is IOException && !(e is DirectoryNotFoundException || + e is PathTooLongException)) { attempt++; - if(attempt > Retries) + if (attempt > Retries) { Log.LogMessage($"Failed to download {uri} to {path} "); return false; } - Log.LogMessage($"Retrying download of '{uri}' to '{path}' due to failure: '{e.Message}' ({attempt}/{Retries})"); + + Log.LogMessage( + $"Retrying download of '{uri}' to '{path}' due to failure: '{e.Message}' ({attempt}/{Retries})"); await Task.Delay(RetryDelayMilliseconds).ConfigureAwait(false); continue; } + } } - } protected async Task HandleBlobPublishingAsync(Dictionary> buildAssets) { @@ -819,26 +808,6 @@ protected async Task HandleBlobPublishingAsync(Dictionary break; } } - else - { - switch (feedConfig.Type) - { - case FeedType.AzDoNugetFeed: - publishTasks.Add(PublishBlobsToAzDoNugetFeedOneByOneAsync(filteredBlobs, buildAssets, - feedConfig)); - break; - case FeedType.AzureStorageFeed: - publishTasks.Add( - PublishBlobsToAzureStorageNugetFeedOneByOneAsync(filteredBlobs, buildAssets, - feedConfig)); - break; - default: - Log.LogError( - $"Unknown target feed type for category '{category}': '{feedConfig.Type}'."); - break; - } - - } } } else @@ -941,20 +910,65 @@ private async Task PublishPackagesToAzDoNugetFeedAsync( Dictionary> buildAssets, TargetFeedConfig feedConfig) { - await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, - async (feed, httpClient, package, feedAccount, feedVisibility, feedName) => + if (!UseApiOverride) + { + await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, + async (feed, httpClient, package, feedAccount, feedVisibility, feedName) => + { + string localPackagePath = + Path.Combine(PackageAssetsBasePath, $"{package.Id}.{package.Version}.nupkg"); + if (!File.Exists(localPackagePath)) + { + Log.LogError($"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; + } + + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); + + await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, package.Version, + feedAccount, feedVisibility, feedName); + }); + } + else + { + string containerId = await GetContainerIdAsync(); + + using HttpClientHandler handler = new HttpClientHandler() { - string localPackagePath = Path.Combine(PackageAssetsBasePath, $"{package.Id}.{package.Version}.nupkg"); - if (!File.Exists(localPackagePath)) + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + CheckCertificateRevocationList = true + }; + string localPackagePath = ""; + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + { + foreach (var package in packagesToPublish) { - Log.LogError($"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); - return; - } + var packageFilename = $"{package.Id}.{package.Version}.nupkg"; + string temporaryPackageDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryPackageDirectory); + localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); + await DownloadFileAsync(client, "PackageArtifacts", containerId, + packageFilename, + localPackagePath); - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); + if (!File.Exists(localPackagePath)) + { + Log.LogError( + $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; + } + + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); + await PushPackageToNugetFeed(feedConfig, localPackagePath, package.Id, package.Version); + File.Delete(localPackagePath); + Directory.Delete(temporaryPackageDirectory); + } + } + } - await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, package.Version, feedAccount, feedVisibility, feedName); - }); } /// @@ -1143,33 +1157,85 @@ private async Task PublishBlobsToAzDoNugetFeedAsync( } } - await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, - async (feed, httpClient, blob, feedAccount, feedVisibility, feedName) => - { - if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.Container)) + if (!UseApiOverride) + { + await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, + async (feed, httpClient, blob, feedAccount, feedVisibility, feedName) => { - // Determine the local path to the blob - string fileName = Path.GetFileName(blob.Id); - string localBlobPath = Path.Combine(BlobAssetsBasePath, fileName); - if (!File.Exists(localBlobPath)) + if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container)) { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); - return; + // Determine the local path to the blob + string fileName = Path.GetFileName(blob.Id); + string localBlobPath = Path.Combine(BlobAssetsBasePath, fileName); + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + return; + } + + string id; + string version; + // Determine package ID and version by asking the nuget libraries + using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) + { + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); + } + + await PushNugetPackageAsync(feed, httpClient, localBlobPath, id, version, feedAccount, + feedVisibility, feedName); } + }); + } + else + { + string containerId = await GetContainerIdAsync(); - string id; - string version; - // Determine package ID and version by asking the nuget libraries - using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) + using HttpClientHandler handler = new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + CheckCertificateRevocationList = true + }; + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) + { + foreach (var blob in blobsToPublish) + { + if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container)) { - PackageIdentity packageIdentity = packageReader.GetIdentity(); - id = packageIdentity.Id; - version = packageIdentity.Version.ToString(); + string temporaryBlobDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryBlobDirectory); + string fileName = Path.GetFileName(blob.Id); + string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); + await DownloadFileAsync(client, "BlobArtifacts", containerId, + fileName, + localBlobPath); + + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + } + + string id; + string version; + using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) + { + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); + } + + await PushBlobToNugetFeed(feedConfig, localBlobPath, id, version); + DeleteTemporaryFile(localBlobPath); + DeleteTemporaryDirectory(temporaryBlobDirectory); } - await PushNugetPackageAsync(feed, httpClient, localBlobPath, id, version, feedAccount, feedVisibility, feedName); } - }); + } + } } private async Task PublishBlobsToAzDoNugetFeedOneByOneAsync( HashSet blobsToPublish, @@ -1192,7 +1258,7 @@ private async Task PublishBlobsToAzDoNugetFeedOneByOneAsync( } } - string containerId = GetContainerId().Result; + string containerId = await GetContainerIdAsync(); using HttpClientHandler handler = new HttpClientHandler() @@ -1278,97 +1344,99 @@ await PushNugetPackageAsync(feedConfig, httpClient, localBlobPath, id, } } } - private async Task PublishPackagesToAzureStorageNugetFeedOneByOneAsync( + + private async Task PublishPackagesToAzureStorageNugetFeedAsync( HashSet packagesToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig) { - string containerId = GetContainerId().Result; - var blobFeedAction = CreateBlobFeedAction(feedConfig); - - var pushOptions = new PushOptions - { - AllowOverwrite = feedConfig.AllowOverwrite, - PassIfExistingItemIdentical = true - }; - - - using HttpClientHandler handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; - string localPackagePath = ""; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + if (UseApiOverride) { - foreach (var package in packagesToPublish) + var packages = packagesToPublish.Select(p => { - string temporaryPackageDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporaryPackageDirectory); - var packageFilename = $"{package.Id}.{package.Version}.nupkg"; - localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, - localPackagePath); - + var localPackagePath = Path.Combine(PackageAssetsBasePath, $"{p.Id}.{p.Version}.nupkg"); if (!File.Exists(localPackagePath)) { - Log.LogError( - $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); - return; + Log.LogError($"Could not locate '{p.Id}.{p.Version}' at '{localPackagePath}'"); } - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.NugetFeed); - IEnumerable item = new List(); - item.ToList().Add(localPackagePath); - await blobFeedAction.PushToFeedAsync(item, pushOptions); - DeleteTemporaryFile(localPackagePath); - DeleteTemporaryDirectory(temporaryPackageDirectory); - } - } - } + return localPackagePath; + }); - private async Task PublishPackagesToAzureStorageNugetFeedAsync( - HashSet packagesToPublish, - Dictionary> buildAssets, - TargetFeedConfig feedConfig) - { - var packages = packagesToPublish.Select(p => - { - var localPackagePath = Path.Combine(PackageAssetsBasePath, $"{p.Id}.{p.Version}.nupkg"); - if (!File.Exists(localPackagePath)) + if (Log.HasLoggedErrors) { - Log.LogError($"Could not locate '{p.Id}.{p.Version}' at '{localPackagePath}'"); + return; } - return localPackagePath; - }); - if (Log.HasLoggedErrors) - { - return; - } + var blobFeedAction = CreateBlobFeedAction(feedConfig); - var blobFeedAction = CreateBlobFeedAction(feedConfig); + var pushOptions = new PushOptions + { + AllowOverwrite = feedConfig.AllowOverwrite, + PassIfExistingItemIdentical = true + }; - var pushOptions = new PushOptions + packagesToPublish + .ToList() + .ForEach(package => TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed)); + + await blobFeedAction.PushToFeedAsync(packages, pushOptions); + } + else { - AllowOverwrite = feedConfig.AllowOverwrite, - PassIfExistingItemIdentical = true - }; + string containerId = await GetContainerIdAsync(); + var blobFeedAction = CreateBlobFeedAction(feedConfig); + + var pushOptions = new PushOptions + { + AllowOverwrite = feedConfig.AllowOverwrite, + PassIfExistingItemIdentical = true + }; + + + using HttpClientHandler handler = new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + CheckCertificateRevocationList = true + }; + string localPackagePath = ""; + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + { + foreach (var package in packagesToPublish) + { + string temporaryPackageDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryPackageDirectory); + var packageFilename = $"{package.Id}.{package.Version}.nupkg"; + localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); + await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, + localPackagePath); - packagesToPublish - .ToList() - .ForEach(package => TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed)); + if (!File.Exists(localPackagePath)) + { + Log.LogError( + $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; + } - await blobFeedAction.PushToFeedAsync(packages, pushOptions); + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); + IEnumerable item = new List(); + item.ToList().Add(localPackagePath); + await blobFeedAction.PushToFeedAsync(item, pushOptions); + DeleteTemporaryFile(localPackagePath); + DeleteTemporaryDirectory(temporaryPackageDirectory); + } + } + } } private async Task PublishBlobsToAzureStorageNugetFeedOneByOneAsync(HashSet blobsToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig) { - string containerId = GetContainerId().Result; + string containerId = await GetContainerIdAsync(); var blobFeedAction = CreateBlobFeedAction(feedConfig); var pushOptions = new PushOptions { @@ -1435,40 +1503,98 @@ private async Task PublishBlobsToAzureStorageNugetFeedAsync( Dictionary> buildAssets, TargetFeedConfig feedConfig) { - var blobs = blobsToPublish - .Select(blob => - { - var fileName = Path.GetFileName(blob.Id); - var localBlobPath = Path.Combine(BlobAssetsBasePath, fileName); - if (!File.Exists(localBlobPath)) + if (!UseApiOverride) + { + var blobs = blobsToPublish + .Select(blob => { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); - } + var fileName = Path.GetFileName(blob.Id); + var localBlobPath = Path.Combine(BlobAssetsBasePath, fileName); + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + } - return new Microsoft.Build.Utilities.TaskItem(localBlobPath, new Dictionary - { - {"RelativeBlobPath", blob.Id} - }); - }) - .ToArray(); + return new Microsoft.Build.Utilities.TaskItem(localBlobPath, new Dictionary + { + {"RelativeBlobPath", blob.Id} + }); + }) + .ToArray(); - if (Log.HasLoggedErrors) - { - return; - } + if (Log.HasLoggedErrors) + { + return; + } - var blobFeedAction = CreateBlobFeedAction(feedConfig); - var pushOptions = new PushOptions + var blobFeedAction = CreateBlobFeedAction(feedConfig); + var pushOptions = new PushOptions + { + AllowOverwrite = feedConfig.AllowOverwrite, + PassIfExistingItemIdentical = true + }; + + blobsToPublish + .ToList() + .ForEach(blob => TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container)); + + await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients: MaxClients, pushOptions); + } + else { - AllowOverwrite = feedConfig.AllowOverwrite, - PassIfExistingItemIdentical = true - }; + string containerId = await GetContainerIdAsync(); + var blobFeedAction = CreateBlobFeedAction(feedConfig); + var pushOptions = new PushOptions + { + AllowOverwrite = feedConfig.AllowOverwrite, + PassIfExistingItemIdentical = true + }; + + using HttpClientHandler handler = new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + CheckCertificateRevocationList = true + }; + using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) + { + { + foreach (var blob in blobsToPublish) + + { + string temporaryBlobDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryBlobDirectory); + var fileName = Path.GetFileName(blob.Id); + var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); + await DownloadFileAsync(client, "BlobArtifacts", containerId, fileName, + localBlobPath); + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + } - blobsToPublish - .ToList() - .ForEach(blob => TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.Container)); + var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, + new Dictionary + { + {"RelativeBlobPath", blob.Id} + }); + + + TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container); + + using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) + { + await blobFeedAction.UploadAssetAsync(item, clientThrottle, pushOptions); + } - await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients: MaxClients, pushOptions); + DeleteTemporaryFile(localBlobPath); + DeleteTemporaryDirectory(temporaryBlobDirectory); + } + } + } + } if (LinkManager == null) { @@ -1518,47 +1644,6 @@ private BlobFeedAction CreateBlobFeedAction(TargetFeedConfig feedConfig) return null; } } - private async Task PublishPackagesToAzDoNugetFeedOneByOneAsync(HashSet packagesToPublish, - Dictionary> buildAssets, - TargetFeedConfig feedConfig) - { - - string containerId = GetContainerId().Result; - - using HttpClientHandler handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; - string localPackagePath = ""; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) - { - foreach (var package in packagesToPublish) - { - var packageFilename = $"{package.Id}.{package.Version}.nupkg"; - string temporaryPackageDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporaryPackageDirectory); - localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - await DownloadFileAsync(client, "PackageArtifacts", containerId, - packageFilename, - localPackagePath); - - if (!File.Exists(localPackagePath)) - { - Log.LogError( - $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); - return; - } - - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.NugetFeed); - await PushPackageToNugetFeed(feedConfig, localPackagePath, package.Id, package.Version); - File.Delete(localPackagePath); - Directory.Delete(temporaryPackageDirectory); - } - } - } private async Task PushPackageToNugetFeed(TargetFeedConfig feedConfig, string localPackagePath, string id , string version) { diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index 4dd62d1dff8..fcbf4259545 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -208,7 +208,7 @@ await Task.WhenAll(new Task[] { HandlePackagePublishingAsync(buildAssets), HandleBlobPublishingAsync(buildAssets), HandleSymbolPublishingAsync(PdbArtifactsBasePath, MsdlToken, - SymWebToken, SymbolPublishingExclusionsFile, PublishSpecialClrFiles,temporarySymbolsLocation) + SymWebToken, SymbolPublishingExclusionsFile, PublishSpecialClrFiles, buildAssets ,temporarySymbolsLocation) }); DeleteTemporaryFiles(temporarySymbolsLocation); diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/BuildArtifacts.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/BuildArtifacts.cs similarity index 100% rename from src/Microsoft.DotNet.Build.Tasks.Feed/src/BuildArtifacts.cs rename to src/Microsoft.DotNet.Build.Tasks.Feed/src/model/BuildArtifacts.cs From ff239e8eb47ad3fad4495c9fd23ee3fa196c10f6 Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Sun, 21 Mar 2021 18:16:43 -0700 Subject: [PATCH 12/32] review comments --- eng/Versions.props | 12 +- global.json | 4 +- .../src/BlobFeedAction.cs | 76 ++- .../src/PublishArtifactsInManifestBase.cs | 577 ++++++------------ 4 files changed, 260 insertions(+), 409 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index fb5df08aa65..f71d399d9de 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -35,8 +35,8 @@ 2.0.0 2.1.1 2.1.0 - 3.9.0-6.21160.10 - 6.0.100-preview.2.21167.1 + 3.9.0-6.21152.10 + 6.0.100-preview.2.21160.1 16.7.1 4.8.3 5.3.0.1 @@ -66,8 +66,8 @@ 2.4.1 2.0.3 2.4.1 - 6.0.0-beta.21168.4 - 6.0.0-beta.21166.5 + 6.0.0-beta.21160.7 + 6.0.0-beta.21160.7 1.22.0 1.1.2 2.0.0 @@ -80,10 +80,10 @@ 1.1.0-beta.20258.6 1.1.0-beta-21151-02 1.1.0-beta-21151-02 - 6.0.0-beta.21166.5 + 6.0.0-beta.21160.7 1.0.0-beta.21153.1 1.1.0-beta.20570.1 - 1.0.0-prerelease.21162.1 + 1.0.0-prerelease.21154.2 1.1.156402 1.1.152002 diff --git a/global.json b/global.json index ba2d76a6d88..38cf1d7ae87 100644 --- a/global.json +++ b/global.json @@ -3,7 +3,7 @@ "dotnet": "6.0.100-preview.2.21155.3" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21168.4", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21166.5" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21160.7", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21160.7" } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs index 77d64e62215..35e8161d23e 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs @@ -1,23 +1,21 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Build.Framework; using Microsoft.DotNet.Build.CloudTestTasks; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Newtonsoft.Json.Serialization; -using NuGet.Packaging; using NuGet.Packaging.Core; using Sleet; -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.IO; -using System.Linq; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; using MSBuild = Microsoft.Build.Utilities; namespace Microsoft.DotNet.Build.Tasks.Feed @@ -140,6 +138,66 @@ await System.Threading.Tasks.Task.WhenAll(taskItems.Select( } } + public async Task UploadAssetAsync( + ITaskItem item, + PushOptions options) + { + string relativeBlobPath = item.GetMetadata("RelativeBlobPath"); + + if (string.IsNullOrEmpty(relativeBlobPath)) + { + string fileName = Path.GetFileName(item.ItemSpec); + string recursiveDir = item.GetMetadata("RecursiveDir"); + relativeBlobPath = $"{recursiveDir}{fileName}"; + } + + relativeBlobPath = $"{RelativePath}{relativeBlobPath}".Replace("\\", "/"); + + if (relativeBlobPath.Contains("//")) + { + Log.LogError( + $"Item '{item.ItemSpec}' RelativeBlobPath contains virtual directory " + + $"without name (double forward slash): '{relativeBlobPath}'"); + return; + } + + Log.LogMessage($"Uploading {relativeBlobPath}"); + try + { + AzureStorageUtils blobUtils = new AzureStorageUtils(AccountName, AccountKey, ContainerName); + + if (!options.AllowOverwrite && await blobUtils.CheckIfBlobExistsAsync(relativeBlobPath)) + { + if (options.PassIfExistingItemIdentical) + { + if (!await blobUtils.IsFileIdenticalToBlobAsync(item.ItemSpec, relativeBlobPath)) + { + Log.LogError( + $"Item '{item}' already exists with different contents " + + $"at '{relativeBlobPath}'"); + } + } + else + { + Log.LogError($"Item '{item}' already exists at '{relativeBlobPath}'"); + } + } + else + { + using (FileStream stream = + new FileStream(item.ItemSpec, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + Log.LogMessage($"Uploading {item} to {relativeBlobPath}."); + await blobUtils.UploadBlockBlobAsync(item.ItemSpec, relativeBlobPath, stream); + } + } + } + catch (Exception exc) + { + Log.LogError($"Unable to upload to {relativeBlobPath} in Azure Storage account {AccountName}/{ContainerName} due to {exc}."); + throw; + } + } public async Task UploadAssetAsync( ITaskItem item, SemaphoreSlim clientThrottle, diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index fdf41c3be4c..5f2c28f56d0 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -15,14 +15,14 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using System.Xml; +using Microsoft.Arcade.Common; using Microsoft.Build.Framework; using Microsoft.DotNet.Build.Tasks.Feed.Model; -using Microsoft.DotNet.Build.Tasks.Feed.src; using Microsoft.DotNet.Maestro.Client; using Microsoft.DotNet.Maestro.Client.Models; using Microsoft.DotNet.VersionTools.BuildManifest.Model; using Newtonsoft.Json; +using NuGet.Packaging; using NuGet.Packaging.Core; using NuGet.Versioning; using static Microsoft.DotNet.Build.Tasks.Feed.GeneralUtils; @@ -164,6 +164,8 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities /// public int RetryDelayMilliseconds { get; set; } = 5000; + private ExponentialRetry RetryHandler = new ExponentialRetry(); + public override bool Execute() { return ExecuteAsync().GetAwaiter().GetResult(); @@ -378,12 +380,7 @@ public async Task HandleSymbolPublishingOneByOneAsync( GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); IEnumerable filesToSymbolServer = null; - using HttpClientHandler handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + using (HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true)) { string localSymbolPath = ""; foreach (var symbol in symbolsToPublish) @@ -394,9 +391,9 @@ public async Task HandleSymbolPublishingOneByOneAsync( localSymbolPath = Path.Combine(temporarySymbDirectory, symbol); await DownloadFileAsync(client, "BlobArtifacts", containerId, symbol, localSymbolPath); - Log.LogMessage($"Local Symbol path to downloaded file {localSymbolPath}"); - IEnumerable symbolFile = new List(); - symbolFile.ToList().Add(localSymbolPath); + Log.LogMessage(MessageImportance.Low, $"Local Symbol path to downloaded file {localSymbolPath}"); + List symbolFiles = new List(); + symbolFiles.ToList().Add(localSymbolPath); foreach (var server in serversToPublish) { var serverPath = server.Key; @@ -407,7 +404,7 @@ await PublishSymbolsHelper.PublishAsync( Log, serverPath, token, - symbolFile, + symbolFiles, filesToSymbolServer, null, ExpirationInDays, @@ -445,7 +442,7 @@ await PublishSymbolsHelper.PublishAsync( if (filesToSymbolServer != null && filesToSymbolServer.Any()) { - IEnumerable symbolFile = null; + List symbolFiles = null; foreach (var server in serversToPublish) { var serverPath = server.Key; @@ -456,7 +453,7 @@ await PublishSymbolsHelper.PublishAsync( Log, serverPath, token, - symbolFile, + symbolFiles, filesToSymbolServer, null, ExpirationInDays, @@ -467,7 +464,11 @@ await PublishSymbolsHelper.PublishAsync( false, true); } - Log.LogMessage($"Successfully published pdb files"); + Log.LogMessage(MessageImportance.High, $"Successfully published pdb files"); + } + else + { + Log.LogMessage(MessageImportance.Low, $"No pdb files to upload to symbol server."); } } @@ -640,15 +641,18 @@ private HashSet FilterPackages(HashSet /// - public async Task GetContainerIdAsync() + private async Task GetContainerIdAsync() { - using (HttpClientHandler handler = new HttpClientHandler { CheckCertificateRevocationList = true }) + string containerId = ""; + using (HttpClient client = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject)) { - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) + string uri = + $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/{AzureProject}/_apis/build/builds/{BuildId}/artifacts?api-version={AzureDevOpsFeedsApiVersion}"; + try { - string uri = - $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/{AzureProject}/_apis/build/builds/{BuildId}/artifacts?api-version={AzureDevOpsFeedsApiVersion}"; - using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); - using HttpResponseMessage response = await client.SendAsync(getMessage); + using HttpResponseMessage response = await client.GetAsync(uri); response.EnsureSuccessStatusCode(); string responseBody = await response.Content.ReadAsStringAsync(); BuildArtifacts buildArtifacts = JsonConvert.DeserializeObject(responseBody); - string containerId = ""; - foreach (var bd in buildArtifacts.value) + + foreach (var artifact in buildArtifacts.value) { - if (string.Equals(bd.name, "BlobArtifacts")) + if (string.Equals(artifact.name, "BlobArtifacts")) { - string[] segment = bd.resource.data.Split('/'); + string[] segment = artifact.resource.data.Split('/'); containerId = segment[1]; break; } } - return containerId; } + catch (Exception e) + { + Log.LogErrorFromException(e); + } + + return containerId; } } - public async Task DownloadFileAsync(HttpClient client, string artifact, string containerId, + public async Task DownloadFileAsync(HttpClient client, string artifact, string containerId, string fileName, string path) { - int attempt = 0; string uri = $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/_apis/resources/Containers/{containerId}?itemPath=/{artifact}/{fileName}&isShallow=true&api-version={AzureApiVersionForFileDownload}"; Log.LogMessage($"download file uri = {uri}"); - while (true) + Exception mostRecentlyCaughtException = null; + bool success = await RetryHandler.RunAsync(async attempt => { try { CancellationTokenSource timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes)); using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); - using HttpResponseMessage response = await client.SendAsync(getMessage, timeoutTokenSource.Token); - if (response.StatusCode == HttpStatusCode.NotFound || - response.ReasonPhrase.StartsWith( - "The requested URI does not represent any resource on the server.", - StringComparison.OrdinalIgnoreCase)) - { - Log.LogMessage( - $"Problems downloading file from '{uri}'. Does the resource exist on the storage? {response.StatusCode} : {response.ReasonPhrase}"); - return false; - } + using HttpResponseMessage response = await client.GetAsync(uri, timeoutTokenSource.Token); response.EnsureSuccessStatusCode(); - using (var fs = new FileStream(path, FileMode.Create, - FileAccess.ReadWrite, FileShare.ReadWrite)) + using var fs = new FileStream(path, FileMode.Create, + FileAccess.ReadWrite, FileShare.ReadWrite); + using var stream = await response.Content.ReadAsStreamAsync(); + + try { - using (var stream = await response.Content.ReadAsStreamAsync()) - { - try - { - await stream.CopyToAsync(fs); - } - finally - { - stream.Close(); - fs.Close(); - } - } + await stream.CopyToAsync(fs); + } + finally + { + stream.Close(); + fs.Close(); } - return true; } - - catch (Exception e) when (e is HttpRequestException || - e is IOException && !(e is DirectoryNotFoundException || - e is PathTooLongException)) - + catch (HttpRequestException toStore) { - attempt++; - if (attempt > Retries) - { - Log.LogMessage($"Failed to download {uri} to {path} "); - return false; - } - - Log.LogMessage( - $"Retrying download of '{uri}' to '{path}' due to failure: '{e.Message}' ({attempt}/{Retries})"); - - await Task.Delay(RetryDelayMilliseconds).ConfigureAwait(false); - continue; + mostRecentlyCaughtException = toStore; + return false; } + }).ConfigureAwait(false); + if (!success) + { + throw new Exception($"Failed to download local file '{path}' after {RetryHandler.MaxAttempts} attempts. See inner exception for details."); } } @@ -933,42 +919,31 @@ await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, pack else { string containerId = await GetContainerIdAsync(); - - using HttpClientHandler handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; string localPackagePath = ""; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true); + foreach (var package in packagesToPublish) { - foreach (var package in packagesToPublish) + var packageFilename = $"{package.Id}.{package.Version}.nupkg"; + string temporaryPackageDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryPackageDirectory); + localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); + await DownloadFileAsync(client, "PackageArtifacts", containerId, + packageFilename, + localPackagePath); + if (!File.Exists(localPackagePath)) { - var packageFilename = $"{package.Id}.{package.Version}.nupkg"; - string temporaryPackageDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporaryPackageDirectory); - localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - await DownloadFileAsync(client, "PackageArtifacts", containerId, - packageFilename, - localPackagePath); - - if (!File.Exists(localPackagePath)) - { - Log.LogError( - $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); - return; - } - - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.NugetFeed); - await PushPackageToNugetFeed(feedConfig, localPackagePath, package.Id, package.Version); - File.Delete(localPackagePath); - Directory.Delete(temporaryPackageDirectory); + Log.LogError( + $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; } + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); + await PushPackageToNugetFeed(feedConfig, localPackagePath, package.Id, package.Version); + File.Delete(localPackagePath); + Directory.Delete(temporaryPackageDirectory); } } - } /// @@ -994,29 +969,30 @@ public async Task PushNugetPackagesAsync(HashSet packagesToPublish, Target string feedVisibility = parsedUri.Groups["visibility"].Value; string feedName = parsedUri.Groups["feed"].Value; - using (var clientThrottle = new SemaphoreSlim(maxClients, maxClients)) + using var clientThrottle = new SemaphoreSlim(maxClients, maxClients); + + using (HttpClient httpClient = new HttpClient(new HttpClientHandler + {CheckCertificateRevocationList = true})) { - using (HttpClient httpClient = new HttpClient(new HttpClientHandler { CheckCertificateRevocationList = true })) - { - httpClient.Timeout = TimeSpan.FromSeconds(180); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( - "Basic", - Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); + httpClient.Timeout = TimeSpan.FromSeconds(180); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); - await Task.WhenAll(packagesToPublish.Select(async packageToPublish => + await Task.WhenAll(packagesToPublish.Select(async packageToPublish => + { + try { - try - { - // Wait to avoid starting too many processes. - await clientThrottle.WaitAsync(); - await packagePublishAction(feedConfig, httpClient, packageToPublish, feedAccount, feedVisibility, feedName); - } - finally - { - clientThrottle.Release(); - } - })); - } + // Wait to avoid starting too many processes. + await clientThrottle.WaitAsync(); + await packagePublishAction(feedConfig, httpClient, packageToPublish, feedAccount, + feedVisibility, feedName); + } + finally + { + clientThrottle.Release(); + } + })); } } @@ -1177,13 +1153,11 @@ await PushNugetPackagesAsync(packagesToPublish, feedConfig, m string id; string version; // Determine package ID and version by asking the nuget libraries - using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) - { - PackageIdentity packageIdentity = packageReader.GetIdentity(); - id = packageIdentity.Id; - version = packageIdentity.Version.ToString(); - } + using var packageReader = new PackageArchiveReader(localBlobPath); + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); await PushNugetPackageAsync(feed, httpClient, localBlobPath, id, version, feedAccount, feedVisibility, feedName); } @@ -1192,82 +1166,7 @@ await PushNugetPackageAsync(feed, httpClient, localBlobPath, id, version, feedAc else { string containerId = await GetContainerIdAsync(); - - using HttpClientHandler handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) - { - foreach (var blob in blobsToPublish) - { - if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.Container)) - { - string temporaryBlobDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporaryBlobDirectory); - string fileName = Path.GetFileName(blob.Id); - string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); - await DownloadFileAsync(client, "BlobArtifacts", containerId, - fileName, - localBlobPath); - - if (!File.Exists(localBlobPath)) - { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); - } - - string id; - string version; - using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) - { - PackageIdentity packageIdentity = packageReader.GetIdentity(); - id = packageIdentity.Id; - version = packageIdentity.Version.ToString(); - } - - await PushBlobToNugetFeed(feedConfig, localBlobPath, id, version); - DeleteTemporaryFile(localBlobPath); - DeleteTemporaryDirectory(temporaryBlobDirectory); - } - - } - } - } - } - private async Task PublishBlobsToAzDoNugetFeedOneByOneAsync( - HashSet blobsToPublish, - Dictionary> buildAssets, - TargetFeedConfig feedConfig) - { - HashSet packagesToPublish = new HashSet(); - - foreach (var blob in blobsToPublish) - { - // Applies to symbol packages and core-sdk's VS feed packages - if (blob.Id.EndsWith(GeneralUtils.PackageSuffix, StringComparison.OrdinalIgnoreCase)) - { - packagesToPublish.Add(blob); - } - else - { - Log.LogWarning( - $"AzDO feed publishing not available for blobs. Blob '{blob.Id}' was not published."); - } - } - - string containerId = await GetContainerIdAsync(); - - - using HttpClientHandler handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) - { + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true, AzureProject); foreach (var blob in blobsToPublish) { if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, @@ -1281,68 +1180,56 @@ private async Task PublishBlobsToAzDoNugetFeedOneByOneAsync( await DownloadFileAsync(client, "BlobArtifacts", containerId, fileName, localBlobPath); - if (!File.Exists(localBlobPath)) { Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); } - string id; string version; - using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) - { - PackageIdentity packageIdentity = packageReader.GetIdentity(); - id = packageIdentity.Id; - version = packageIdentity.Version.ToString(); - } + using var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath); + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); await PushBlobToNugetFeed(feedConfig, localBlobPath, id, version); DeleteTemporaryFile(localBlobPath); DeleteTemporaryDirectory(temporaryBlobDirectory); } - } } - } - - private async Task PushBlobToNugetFeed(TargetFeedConfig feedConfig, string localBlobPath, string id , string version) + private async Task PushBlobToNugetFeed(TargetFeedConfig feedConfig, string localBlobPath, string id, + string version) { var parsedUri = Regex.Match(feedConfig.TargetURL, PublishingConstants.AzDoNuGetFeedPattern); if (!parsedUri.Success) { - Log.LogError($"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); + Log.LogError( + $"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); return; } string feedAccount = parsedUri.Groups["account"].Value; string feedVisibility = parsedUri.Groups["visibility"].Value; string feedName = parsedUri.Groups["feed"].Value; - using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) + using HttpClient httpClient = new HttpClient(new HttpClientHandler + {CheckCertificateRevocationList = true}); + httpClient.Timeout = TimeSpan.FromSeconds(180); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String( + Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", + feedConfig.Token)))); + try { - using (HttpClient httpClient = new HttpClient(new HttpClientHandler - { CheckCertificateRevocationList = true })) - { - httpClient.Timeout = TimeSpan.FromSeconds(180); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( - "Basic", - Convert.ToBase64String( - Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", - feedConfig.Token)))); - try - { - // Wait to avoid starting too many processes. - await clientThrottle.WaitAsync(); - await PushNugetPackageAsync(feedConfig, httpClient, localBlobPath, id, - version, feedAccount, feedVisibility, feedName); - } - finally - { - clientThrottle.Release(); - } - - } + await PushNugetPackageAsync(feedConfig, httpClient, localBlobPath, id, + version, feedAccount, feedVisibility, feedName); } + catch (Exception e) + { + Log.LogErrorFromException(e); + } + } private async Task PublishPackagesToAzureStorageNugetFeedAsync( @@ -1393,110 +1280,33 @@ private async Task PublishPackagesToAzureStorageNugetFeedAsync( AllowOverwrite = feedConfig.AllowOverwrite, PassIfExistingItemIdentical = true }; - - - using HttpClientHandler handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; string localPackagePath = ""; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg)) - { - foreach (var package in packagesToPublish) - { - string temporaryPackageDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporaryPackageDirectory); - var packageFilename = $"{package.Id}.{package.Version}.nupkg"; - localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, - localPackagePath); - - if (!File.Exists(localPackagePath)) - { - Log.LogError( - $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); - return; - } + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true); - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.NugetFeed); - IEnumerable item = new List(); - item.ToList().Add(localPackagePath); - await blobFeedAction.PushToFeedAsync(item, pushOptions); - DeleteTemporaryFile(localPackagePath); - DeleteTemporaryDirectory(temporaryPackageDirectory); - } - } - } - } - - private async Task PublishBlobsToAzureStorageNugetFeedOneByOneAsync(HashSet blobsToPublish, - Dictionary> buildAssets, - TargetFeedConfig feedConfig) - { - string containerId = await GetContainerIdAsync(); - var blobFeedAction = CreateBlobFeedAction(feedConfig); - var pushOptions = new PushOptions - { - AllowOverwrite = feedConfig.AllowOverwrite, - PassIfExistingItemIdentical = true - }; - - using HttpClientHandler handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) - { + foreach (var package in packagesToPublish) { - foreach (var blob in blobsToPublish) - + string temporaryPackageDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryPackageDirectory); + var packageFilename = $"{package.Id}.{package.Version}.nupkg"; + localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); + await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, + localPackagePath); + if (!File.Exists(localPackagePath)) { - string temporaryBlobDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporaryBlobDirectory); - var fileName = Path.GetFileName(blob.Id); - var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); - await DownloadFileAsync(client, "BlobArtifacts", containerId, fileName, - localBlobPath); - if (!File.Exists(localBlobPath)) - { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); - } - - var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, - new Dictionary - { - {"RelativeBlobPath", blob.Id} - }); - - - TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.Container); - - using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) - { - await blobFeedAction.UploadAssetAsync(item, clientThrottle, pushOptions); - } - - DeleteTemporaryFile(localBlobPath); - DeleteTemporaryDirectory(temporaryBlobDirectory); + Log.LogError( + $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; } + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); + IEnumerable item = new List(); + item.ToList().Add(localPackagePath); + await blobFeedAction.PushToFeedAsync(item, pushOptions); + DeleteTemporaryFile(localPackagePath); + DeleteTemporaryDirectory(temporaryPackageDirectory); } } - if (LinkManager == null) - { - LinkManager = new LatestLinksManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, AkaMSGroupOwner, - AkaMSCreatedBy, AkaMsOwners, Log); - } - - // The latest links should be updated only after the publishing is complete, to avoid - // dead links in the interim. - await LinkManager.CreateOrUpdateLatestLinksAsync(blobsToPublish, feedConfig, - PublishingConstants.ExpectedFeedUrlSuffix.Length); } private async Task PublishBlobsToAzureStorageNugetFeedAsync( HashSet blobsToPublish, @@ -1550,49 +1360,31 @@ private async Task PublishBlobsToAzureStorageNugetFeedAsync( AllowOverwrite = feedConfig.AllowOverwrite, PassIfExistingItemIdentical = true }; + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true, AzureProject); - using HttpClientHandler handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - CheckCertificateRevocationList = true - }; - using (HttpClient client = CreateHttpClient(handler, AzureDevOpsOrg, AzureProject)) + foreach (var blob in blobsToPublish) { + string temporaryBlobDirectory = + Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryBlobDirectory); + var fileName = Path.GetFileName(blob.Id); + var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); + await DownloadFileAsync(client, "BlobArtifacts", containerId, fileName, + localBlobPath); + if (!File.Exists(localBlobPath)) { - foreach (var blob in blobsToPublish) - - { - string temporaryBlobDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporaryBlobDirectory); - var fileName = Path.GetFileName(blob.Id); - var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); - await DownloadFileAsync(client, "BlobArtifacts", containerId, fileName, - localBlobPath); - if (!File.Exists(localBlobPath)) - { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); - } - - var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, - new Dictionary - { - {"RelativeBlobPath", blob.Id} - }); - - - TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.Container); - - using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) - { - await blobFeedAction.UploadAssetAsync(item, clientThrottle, pushOptions); - } - - DeleteTemporaryFile(localBlobPath); - DeleteTemporaryDirectory(temporaryBlobDirectory); - } + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); } + var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, + new Dictionary + { + {"RelativeBlobPath", blob.Id} + }); + TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container); + await blobFeedAction.UploadAssetAsync(item, pushOptions); + DeleteTemporaryFile(localBlobPath); + DeleteTemporaryDirectory(temporaryBlobDirectory); } } @@ -1645,7 +1437,8 @@ private BlobFeedAction CreateBlobFeedAction(TargetFeedConfig feedConfig) } } - private async Task PushPackageToNugetFeed(TargetFeedConfig feedConfig, string localPackagePath, string id , string version) + private async Task PushPackageToNugetFeed(TargetFeedConfig feedConfig, string localPackagePath, string id, + string version) { var parsedUri = Regex.Match(feedConfig.TargetURL, PublishingConstants.AzDoNuGetFeedPattern); if (!parsedUri.Success) @@ -1659,18 +1452,18 @@ private async Task PushPackageToNugetFeed(TargetFeedConfig feedConfig, string lo string feedVisibility = parsedUri.Groups["visibility"].Value; string feedName = parsedUri.Groups["feed"].Value; - using (HttpClient httpClient = new HttpClient(new HttpClientHandler - {CheckCertificateRevocationList = true})) - { - httpClient.Timeout = TimeSpan.FromSeconds(180); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( - "Basic", - Convert.ToBase64String( - Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); - await PushNugetPackageAsync(feedConfig, httpClient, localPackagePath, id, - version, - feedAccount, feedVisibility, feedName); - } + using HttpClient httpClient = new HttpClient(new HttpClientHandler + {CheckCertificateRevocationList = true}); + + httpClient.Timeout = TimeSpan.FromSeconds(180); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String( + Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); + await PushNugetPackageAsync(feedConfig, httpClient, localPackagePath, id, + version, + feedAccount, feedVisibility, feedName); + } /// From 696ae339169e114e79d822d1413265d512b7b55b Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Sun, 21 Mar 2021 18:28:58 -0700 Subject: [PATCH 13/32] test --- eng/Versions.props | 5 ----- 1 file changed, 5 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index da436b5b428..e7f5b17c8a5 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -35,13 +35,8 @@ 2.0.0 2.1.1 2.1.0 -<<<<<<< HEAD - 3.9.0-6.21152.10 - 6.0.100-preview.2.21160.1 -======= 3.9.0-6.21160.10 6.0.100-preview.2.21169.1 ->>>>>>> 0d3414187657bcc31ff4b02f34d9fbd0e85e18dc 16.7.1 4.8.3 5.3.0.1 From 3cc26df330434807c21574b1439976f1245477d7 Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Mon, 22 Mar 2021 18:24:19 -0700 Subject: [PATCH 14/32] review comments: --- .../src/BlobFeedAction.cs | 2 +- .../src/PublishArtifactsInManifestBase.cs | 17 +++++++++++++---- .../src/PublishArtifactsInManifestV3.cs | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs index 35e8161d23e..2e0c068d5d3 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs @@ -390,7 +390,7 @@ private ISleetFileSystem GetAzureFileSystem(LocalCache fileCache) } } - private async Task PushAsync(IEnumerable items, PushOptions options) + public async Task PushAsync(IEnumerable items, PushOptions options) { LocalSettings settings = GetSettings(); SleetLogger log = new SleetLogger(Log, NuGet.Common.LogLevel.Verbose); diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 5f2c28f56d0..381d456a6a4 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -1186,7 +1186,7 @@ await DownloadFileAsync(client, "BlobArtifacts", containerId, } string id; string version; - using var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath); + using var packageReader = new PackageArchiveReader(localBlobPath); PackageIdentity packageIdentity = packageReader.GetIdentity(); id = packageIdentity.Id; @@ -1300,9 +1300,18 @@ await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename } TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); - IEnumerable item = new List(); - item.ToList().Add(localPackagePath); - await blobFeedAction.PushToFeedAsync(item, pushOptions); + IEnumerable items = new List(); + items.ToList().Add(localPackagePath); + Log.LogMessage(MessageImportance.Low, $"START pushing {packageFilename} to feed"); + try + { + await blobFeedAction.PushAsync(items, pushOptions); + } + catch (Exception e) + { + Log.LogErrorFromException(e); + } + DeleteTemporaryFile(localPackagePath); DeleteTemporaryDirectory(temporaryPackageDirectory); } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index fcbf4259545..6ee2b38df8f 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. + // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.Build.Framework; From cf46b2dbff7ff6861da611b605d0f6ca1a8ebc51 Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Mon, 22 Mar 2021 20:15:06 -0700 Subject: [PATCH 15/32] minor fix --- .../src/PublishArtifactsInManifestBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 381d456a6a4..8d8ddc016eb 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -643,7 +643,7 @@ private HashSet FilterPackages(HashSet Date: Tue, 23 Mar 2021 00:51:46 -0700 Subject: [PATCH 16/32] minor fix --- .../PublishToSymbolServerTest.cs | 4 +- .../src/PublishArtifactsInManifestBase.cs | 122 ++++++++++-------- 2 files changed, 70 insertions(+), 56 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs index a9bcf5f5cbc..4c0b105e34c 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs @@ -3,6 +3,7 @@ using Castle.DynamicProxy.Generators.Emitters.SimpleAST; using Microsoft.Arcade.Test.Common; using Microsoft.DotNet.Build.Tasks.Feed.Model; +using Microsoft.DotNet.Maestro.Client.Models; using Xunit; namespace Microsoft.DotNet.Build.Tasks.Feed.Tests @@ -85,7 +86,8 @@ public void TemporarySymbolDirectoryDoesNotExists() BuildEngine = buildEngine, }; var path = TestInputs.GetFullPath("Symbol"); - //var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, path); + var buildAsset = new Dictionary>(); + var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, buildAsset, path); Assert.True(task.Log.HasLoggedErrors); } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 8d8ddc016eb..f2ed7638732 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -343,6 +343,17 @@ public void CheckForStableAssetsInNonIsolatedFeeds() } } } + + /// + /// Publishes files to symbol server(s) one by one using Azure api to download files + /// + /// + /// + /// + /// + /// + /// + /// public async Task HandleSymbolPublishingOneByOneAsync( string pdbArtifactsBasePath, string msdlToken, @@ -351,17 +362,12 @@ public async Task HandleSymbolPublishingOneByOneAsync( bool publishSpecialClrFiles, Dictionary> buildAssets) { - StringBuilder symbolLog = new StringBuilder(); symbolLog.AppendLine("Publishing Symbols to Symbol server: "); - var symbolCategory = TargetFeedContentType.Symbols; - string containerId = await GetContainerIdAsync(); - HashSet symbolsToPublish = new HashSet(); //Get all the symbol file names - foreach (var asset in buildAssets) { var name = asset.Key; @@ -373,9 +379,7 @@ public async Task HandleSymbolPublishingOneByOneAsync( } } - HashSet feedConfigsForSymbols = FeedConfigs[symbolCategory]; - Dictionary serversToPublish = GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); @@ -415,11 +419,9 @@ await PublishSymbolsHelper.PublishAsync( false, true); } - DeleteTemporaryFile(localSymbolPath); DeleteTemporaryDirectory(temporarySymbDirectory); } - symbolLog.AppendLine( $"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false "); symbolLog.Append($"\nTotal number of symbol files : {symbolsToPublish.Count}"); @@ -427,9 +429,7 @@ await PublishSymbolsHelper.PublishAsync( symbolLog.AppendLine(); Log.LogMessage(MessageImportance.High, symbolLog.ToString()); symbolLog.Clear(); - } - // publishing pdb artifacts if (Directory.Exists(pdbArtifactsBasePath)) { @@ -495,48 +495,57 @@ public async Task HandleSymbolPublishingAsync ( { StringBuilder symbolLog = new StringBuilder(); symbolLog.AppendLine("Publishing Symbols to Symbol server: "); - string[] fileEntries = Directory.GetFiles(temporarySymbolsLocation); + if (Directory.Exists(temporarySymbolsLocation)) + { + string[] fileEntries = Directory.GetFiles(temporarySymbolsLocation); - var category = TargetFeedContentType.Symbols; + var category = TargetFeedContentType.Symbols; - HashSet feedConfigsForSymbols = FeedConfigs[category]; + HashSet feedConfigsForSymbols = FeedConfigs[category]; - Dictionary serversToPublish = - GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); + Dictionary serversToPublish = + GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); - IEnumerable filesToSymbolServer = null; - if (Directory.Exists(pdbArtifactsBasePath)) - { - var pdbEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.pdb", System.IO.SearchOption.AllDirectories); - var dllEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.dll", System.IO.SearchOption.AllDirectories); - filesToSymbolServer = pdbEntries.Concat(dllEntries); - } + IEnumerable filesToSymbolServer = null; + if (Directory.Exists(pdbArtifactsBasePath)) + { + var pdbEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.pdb", + System.IO.SearchOption.AllDirectories); + var dllEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.dll", + System.IO.SearchOption.AllDirectories); + filesToSymbolServer = pdbEntries.Concat(dllEntries); + } - foreach (var server in serversToPublish) + foreach (var server in serversToPublish) + { + var serverPath = server.Key; + var token = server.Value; + symbolLog.AppendLine($"Publishing symbol packages to {serverPath}:"); + symbolLog.AppendLine( + $"Performing symbol publishing...\nSymbolServerPath : ${serverPath} \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false \nTotal number of symbol files : {fileEntries.Length} "); + await PublishSymbolsHelper.PublishAsync( + Log, + serverPath, + token, + fileEntries, + filesToSymbolServer, + null, + ExpirationInDays, + false, + publishSpecialClrFiles, + null, + false, + false, + true); + symbolLog.AppendLine("Successfully published to Symbol Server."); + symbolLog.AppendLine(); + Log.LogMessage(MessageImportance.High, symbolLog.ToString()); + symbolLog.Clear(); + } + } + else { - var serverPath = server.Key; - var token = server.Value; - symbolLog.AppendLine($"Publishing symbol packages to {serverPath}:"); - symbolLog.AppendLine( - $"Performing symbol publishing...\nSymbolServerPath : ${serverPath} \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false \nTotal number of symbol files : {fileEntries.Length} "); - await PublishSymbolsHelper.PublishAsync( - Log, - serverPath, - token, - fileEntries, - filesToSymbolServer, - null, - ExpirationInDays, - false, - publishSpecialClrFiles, - null, - false, - false, - true); - symbolLog.AppendLine("Successfully published to Symbol Server."); - symbolLog.AppendLine(); - Log.LogMessage(MessageImportance.High, symbolLog.ToString()); - symbolLog.Clear(); + Log.LogError("Temporary symbols directory does not exists."); } } else @@ -599,7 +608,6 @@ protected async Task HandlePackagePublishingAsync(Dictionary GetContainerIdAsync() { Log.LogErrorFromException(e); } - return containerId; } } - public async Task DownloadFileAsync(HttpClient client, string artifact, string containerId, + /// + /// Download artifact file using Azure API + /// + /// + /// + /// + /// + /// + /// + private async Task DownloadFileAsync(HttpClient client, string artifact, string containerId, string fileName, string path) { string uri = @@ -801,7 +815,6 @@ protected async Task HandleBlobPublishingAsync(Dictionary Log.LogError($"No target feed configuration found for artifact category: '{category}'."); } } - await Task.WhenAll(publishTasks); } @@ -1311,12 +1324,12 @@ await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename { Log.LogErrorFromException(e); } - DeleteTemporaryFile(localPackagePath); DeleteTemporaryDirectory(temporaryPackageDirectory); } } } + private async Task PublishBlobsToAzureStorageNugetFeedAsync( HashSet blobsToPublish, Dictionary> buildAssets, @@ -1472,7 +1485,6 @@ private async Task PushPackageToNugetFeed(TargetFeedConfig feedConfig, string lo await PushNugetPackageAsync(feedConfig, httpClient, localPackagePath, id, version, feedAccount, feedVisibility, feedName); - } /// From 16826fb607659039d014980c51d490570251e12c Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Wed, 24 Mar 2021 12:19:33 -0700 Subject: [PATCH 17/32] review: --- eng/publishing/v3/publish-assets.yml | 2 +- .../SdkTasks/PublishArtifactsInManifest.proj | 2 +- .../src/PublishArtifactsInManifest.cs | 6 +- .../src/PublishArtifactsInManifestBase.cs | 55 +++++++++++++------ 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml index 3d78cacb56d..c360f7b9231 100644 --- a/eng/publishing/v3/publish-assets.yml +++ b/eng/publishing/v3/publish-assets.yml @@ -107,7 +107,7 @@ jobs: /p:SymWebToken=$(symweb-symbol-server-pat) /p:BuildQuality='${{ parameters.buildQuality }}' /p:AzdoApiToken='$(dn-bot-dotnet-build-rw-code-rw)' - /p:StagingDir='$(Build.ArtifactStagingDirectory)/' + /p:ArtifactsBasePath='$(Build.ArtifactStagingDirectory)/' /p:BuildId='$(AzDOBuildId)' /p:AzureDevOpsOrg='dnceng' /p:AzureProject='$(AzDOProjectName)' diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index 7f94a16b257..c7229599643 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -140,7 +140,7 @@ ShippingFeedOverride="$(ShippingFeedOverride)" SymbolsFeedOverride="$(SymbolsFeedOverride)" AzdoApiToken="$(AzdoApiToken)" - StagingDir="$(StagingDir)" + ArtifactsBasePath="$(ArtifactsBasePath)" BuildId="$(BuildId)" AzureDevOpsOrg="$(AzureDevOpsOrg)" AzureProject="$(AzureProject)" diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs index 99b8fc473e4..3b433722960 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs @@ -185,7 +185,7 @@ public string BuildQuality } public string AzdoApiToken {get; set;} - public string StagingDir { get; set;} + public string ArtifactsBasePath { get; set;} public string BuildId { get; set; } @@ -193,7 +193,7 @@ public string BuildQuality public string AzureDevOpsOrg { get; set; } - public bool UseApiOverride {get; set;} + public bool UseApiOverride { get; set; } = false; /// /// Just an internal flag to keep track whether we published assets via a V3 manifest or not. @@ -384,7 +384,7 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui ShippingFeedOverride = this.ShippingFeedOverride, TransportFeedOverride = this.TransportFeedOverride, SymbolsFeedOverride = this.SymbolsFeedOverride, - StagingDir = this.StagingDir, + ArtifactsBasePath = this.ArtifactsBasePath, AzdoApiToken = this.AzdoApiToken, BuildId = this.BuildId, AzureProject = this.AzureProject, diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index f2ed7638732..59fe9f4a702 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -109,7 +109,7 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities public string AzdoApiToken { get; set; } - public string StagingDir { get; set; } + public string ArtifactsBasePath { get; set; } public string AzureDevOpsFeedsApiVersion { get; set; } = "6.0"; @@ -389,10 +389,10 @@ public async Task HandleSymbolPublishingOneByOneAsync( string localSymbolPath = ""; foreach (var symbol in symbolsToPublish) { - string temporarySymbDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporarySymbDirectory); - localSymbolPath = Path.Combine(temporarySymbDirectory, symbol); + string temporarySymbolsDirectory = CreateTemporaryDirectory(); + //Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); + //EnsureTemporaryDirectoryExists(temporarySymbolsDirectory); + localSymbolPath = Path.Combine(temporarySymbolsDirectory, symbol); await DownloadFileAsync(client, "BlobArtifacts", containerId, symbol, localSymbolPath); Log.LogMessage(MessageImportance.Low, $"Local Symbol path to downloaded file {localSymbolPath}"); @@ -420,7 +420,7 @@ await PublishSymbolsHelper.PublishAsync( true); } DeleteTemporaryFile(localSymbolPath); - DeleteTemporaryDirectory(temporarySymbDirectory); + DeleteTemporaryDirectory(temporarySymbolsDirectory); } symbolLog.AppendLine( $"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false "); @@ -705,6 +705,11 @@ private async Task GetContainerIdAsync() { Log.LogErrorFromException(e); } + + if (string.IsNullOrEmpty(containerId)) + { + Log.LogError("Container Id does not exists"); + } return containerId; } } @@ -938,7 +943,7 @@ await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, pack { var packageFilename = $"{package.Id}.{package.Version}.nupkg"; string temporaryPackageDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); + Path.GetFullPath(Path.Combine(ArtifactsBasePath,Guid.NewGuid().ToString())); EnsureTemporaryDirectoryExists(temporaryPackageDirectory); localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); await DownloadFileAsync(client, "PackageArtifacts", containerId, @@ -1185,9 +1190,9 @@ await PushNugetPackageAsync(feed, httpClient, localBlobPath, id, version, feedAc if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.Container)) { - string temporaryBlobDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporaryBlobDirectory); + string temporaryBlobDirectory = CreateTemporaryDirectory(); + //Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); + //EnsureTemporaryDirectoryExists(temporaryBlobDirectory); string fileName = Path.GetFileName(blob.Id); string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); await DownloadFileAsync(client, "BlobArtifacts", containerId, @@ -1259,7 +1264,6 @@ private async Task PublishPackagesToAzureStorageNugetFeedAsync( { Log.LogError($"Could not locate '{p.Id}.{p.Version}' at '{localPackagePath}'"); } - return localPackagePath; }); @@ -1298,9 +1302,9 @@ private async Task PublishPackagesToAzureStorageNugetFeedAsync( foreach (var package in packagesToPublish) { - string temporaryPackageDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporaryPackageDirectory); + string temporaryPackageDirectory = CreateTemporaryDirectory(); + //Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); + //EnsureTemporaryDirectoryExists(temporaryPackageDirectory); var packageFilename = $"{package.Id}.{package.Version}.nupkg"; localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, @@ -1330,6 +1334,14 @@ await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename } } + public string CreateTemporaryDirectory() + { + string temporaryDirectory = + Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryDirectory); + return temporaryDirectory; + } + private async Task PublishBlobsToAzureStorageNugetFeedAsync( HashSet blobsToPublish, Dictionary> buildAssets, @@ -1386,9 +1398,9 @@ private async Task PublishBlobsToAzureStorageNugetFeedAsync( foreach (var blob in blobsToPublish) { - string temporaryBlobDirectory = - Path.GetFullPath(Path.Combine(StagingDir, @"..\", Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporaryBlobDirectory); + string temporaryBlobDirectory = CreateTemporaryDirectory(); + //Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); + //EnsureTemporaryDirectoryExists(temporaryBlobDirectory); var fileName = Path.GetFileName(blob.Id); var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); await DownloadFileAsync(client, "BlobArtifacts", containerId, fileName, @@ -1604,6 +1616,15 @@ protected bool AnyMissingRequiredBaseProperties() Log.LogError($"The property {nameof(BuildAssetRegistryToken)} is required but doesn't have a value set."); } + if (string.IsNullOrEmpty(AzdoApiToken)) + { + Log.LogError($"The property {nameof(AzdoApiToken)} is required but doesn't have a value set."); + } + + if (string.IsNullOrEmpty(ArtifactsBasePath)) + { + Log.LogError($"The property {nameof(ArtifactsBasePath)} is required but doesn't have a value set."); + } return Log.HasLoggedErrors; } } From 2a3eda7092db6f3cc4902656b08c920e1bde1413 Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 25 Mar 2021 12:07:02 -0700 Subject: [PATCH 18/32] review comments --- .../tools/SdkTasks/PublishArtifactsInManifest.proj | 4 ++++ .../src/PublishArtifactsInManifest.cs | 4 ++++ .../src/PublishArtifactsInManifestBase.cs | 10 ++++------ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index c7229599643..881a6b4c9c1 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -23,6 +23,10 @@ - SymWebToken : Token used to publish symbol blobs, dll and pdb to Symweb - SymbolPublishingExclusionsFile : Files that have to be excluded from symbol publishing - PdbArtifactsBasePath : Path to dlls and pdbs + - AzdoApiToken : Token used to call the Azure Api to download artifacts and symbols + - ArtifactsBasePath : Staging directory + - UseApiOverride : When set to true it will use Azure Api to download the artifacts and symbols one file at a time + during publishing, else all the artifacts and symbols are downloaded before publishing. Optional aka.ms link generation parameters: - AkaMSClientId : Client ID for the aka.ms AD application diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs index 3b433722960..b26281ef3a6 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs @@ -193,6 +193,10 @@ public string BuildQuality public string AzureDevOpsOrg { get; set; } + /// + /// If true, uses Azdo Api to download artifacts and symbols files one file at a time during publishing process. + /// If it is set to false, then artifacts and symbols are downloaded in PackageArtifacts and BlobArtifacts directory before publishing. + /// public bool UseApiOverride { get; set; } = false; /// diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 59fe9f4a702..43b24cec106 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -390,8 +390,6 @@ public async Task HandleSymbolPublishingOneByOneAsync( foreach (var symbol in symbolsToPublish) { string temporarySymbolsDirectory = CreateTemporaryDirectory(); - //Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); - //EnsureTemporaryDirectoryExists(temporarySymbolsDirectory); localSymbolPath = Path.Combine(temporarySymbolsDirectory, symbol); await DownloadFileAsync(client, "BlobArtifacts", containerId, symbol, localSymbolPath); @@ -1191,8 +1189,6 @@ await PushNugetPackageAsync(feed, httpClient, localBlobPath, id, version, feedAc AddAssetLocationToAssetAssetLocationType.Container)) { string temporaryBlobDirectory = CreateTemporaryDirectory(); - //Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); - //EnsureTemporaryDirectoryExists(temporaryBlobDirectory); string fileName = Path.GetFileName(blob.Id); string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); await DownloadFileAsync(client, "BlobArtifacts", containerId, @@ -1303,8 +1299,6 @@ private async Task PublishPackagesToAzureStorageNugetFeedAsync( foreach (var package in packagesToPublish) { string temporaryPackageDirectory = CreateTemporaryDirectory(); - //Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); - //EnsureTemporaryDirectoryExists(temporaryPackageDirectory); var packageFilename = $"{package.Id}.{package.Version}.nupkg"; localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, @@ -1334,6 +1328,10 @@ await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename } } + /// + /// Creates a temporary directory + /// + /// public string CreateTemporaryDirectory() { string temporaryDirectory = From 0cada85e0b0cedef7708434cc405cd33197c660f Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Mon, 29 Mar 2021 19:20:47 -0700 Subject: [PATCH 19/32] Review comments --- eng/publishing/v3/publish-assets.yml | 6 +- .../SdkTasks/PublishArtifactsInManifest.proj | 6 +- .../PublishToSymbolServerTest.cs | 2 - .../src/PublishArtifactsInManifest.cs | 4 +- .../src/PublishArtifactsInManifestBase.cs | 702 ++++++++++-------- .../src/PublishArtifactsInManifestV3.cs | 14 +- 6 files changed, 427 insertions(+), 307 deletions(-) diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml index c360f7b9231..0a08d5f2463 100644 --- a/eng/publishing/v3/publish-assets.yml +++ b/eng/publishing/v3/publish-assets.yml @@ -106,12 +106,12 @@ jobs: /p:MsdlToken=$(microsoft-symbol-server-pat) /p:SymWebToken=$(symweb-symbol-server-pat) /p:BuildQuality='${{ parameters.buildQuality }}' - /p:AzdoApiToken='$(dn-bot-dotnet-build-rw-code-rw)' + /p:AzdoApiToken='$(dn-bot-all-orgs-build-rw-code-rw)' /p:ArtifactsBasePath='$(Build.ArtifactStagingDirectory)/' /p:BuildId='$(AzDOBuildId)' - /p:AzureDevOpsOrg='dnceng' + /p:AzureDevOpsOrg='$(AzDOAccount)' /p:AzureProject='$(AzDOProjectName)' - /p:UseApiOverride='true' + /p:UseStreamingPublishing='true' - template: /eng/common/templates/steps/publish-logs.yml parameters: StageLabel: '${{ parameters.stageName }}' diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index 881a6b4c9c1..103b43554f2 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -25,8 +25,8 @@ - PdbArtifactsBasePath : Path to dlls and pdbs - AzdoApiToken : Token used to call the Azure Api to download artifacts and symbols - ArtifactsBasePath : Staging directory - - UseApiOverride : When set to true it will use Azure Api to download the artifacts and symbols one file at a time - during publishing, else all the artifacts and symbols are downloaded before publishing. + - UseStreamingPublishing : When set to true it will use Azure Api to download the artifacts and symbols using streaming, + else all the artifacts and symbols are downloaded before publishing. Optional aka.ms link generation parameters: - AkaMSClientId : Client ID for the aka.ms AD application @@ -148,7 +148,7 @@ BuildId="$(BuildId)" AzureDevOpsOrg="$(AzureDevOpsOrg)" AzureProject="$(AzureProject)" - UseApiOverride="$(UseApiOverride)"/> + UseStreamingPublishing="$(UseStreamingPublishing)"/> diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs index 4c0b105e34c..0133c110406 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs @@ -1,6 +1,5 @@ using System.Collections.Generic; using System.IO; -using Castle.DynamicProxy.Generators.Emitters.SimpleAST; using Microsoft.Arcade.Test.Common; using Microsoft.DotNet.Build.Tasks.Feed.Model; using Microsoft.DotNet.Maestro.Client.Models; @@ -136,6 +135,5 @@ public void PublishSymbolApiIsCalledTest() false, false).IsCompleted); } - } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs index b26281ef3a6..0f5be881c36 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs @@ -197,7 +197,7 @@ public string BuildQuality /// If true, uses Azdo Api to download artifacts and symbols files one file at a time during publishing process. /// If it is set to false, then artifacts and symbols are downloaded in PackageArtifacts and BlobArtifacts directory before publishing. /// - public bool UseApiOverride { get; set; } = false; + public bool UseStreamingPublishing { get; set; } = false; /// /// Just an internal flag to keep track whether we published assets via a V3 manifest or not. @@ -393,7 +393,7 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui BuildId = this.BuildId, AzureProject = this.AzureProject, AzureDevOpsOrg = this.AzureDevOpsOrg, - UseApiOverride = this.UseApiOverride + UseStreamingPublishing = this.UseStreamingPublishing }; } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 43b24cec106..eb65e95ea4f 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -123,7 +123,7 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities private readonly string AzureDevOpsBaseUrl = $"https://dev.azure.com"; - public bool UseApiOverride { get; set; } + public bool UseStreamingPublishing { get; set; } public int Retries = 5; @@ -164,7 +164,11 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities /// public int RetryDelayMilliseconds { get; set; } = 5000; - private ExponentialRetry RetryHandler = new ExponentialRetry(); + public ExponentialRetry RetryHandler = new ExponentialRetry + { + MaxAttempts = 5, + DelayBase = 2.5 // 2.5 ^ 5 = ~1.5 minutes max wait between retries + }; public override bool Execute() { @@ -354,7 +358,7 @@ public void CheckForStableAssetsInNonIsolatedFeeds() /// /// /// - public async Task HandleSymbolPublishingOneByOneAsync( + public async Task PublishSymbolsUsingStreamingAsync( string pdbArtifactsBasePath, string msdlToken, string symWebToken, @@ -373,10 +377,7 @@ public async Task HandleSymbolPublishingOneByOneAsync( var name = asset.Key; if (name.Contains(".symbols.nupkg")) { - var segments = name.Split('/'); - var fileName = segments[segments.Length - 1]; - symbolsToPublish.Add(fileName); - + symbolsToPublish.Add(Path.GetFileName(name)); } } HashSet feedConfigsForSymbols = FeedConfigs[symbolCategory]; @@ -401,21 +402,27 @@ public async Task HandleSymbolPublishingOneByOneAsync( var serverPath = server.Key; var token = server.Value; symbolLog.AppendLine($"Publishing symbol file {symbol} to {serverPath}:"); - - await PublishSymbolsHelper.PublishAsync( - Log, - serverPath, - token, - symbolFiles, - filesToSymbolServer, - null, - ExpirationInDays, - false, - publishSpecialClrFiles, - null, - false, - false, - true); + try + { + await PublishSymbolsHelper.PublishAsync( + Log, + serverPath, + token, + symbolFiles, + filesToSymbolServer, + null, + ExpirationInDays, + false, + publishSpecialClrFiles, + null, + false, + false, + true); + } + catch (Exception ex) + { + Log.LogError(ex.Message); + } } DeleteTemporaryFile(localSymbolPath); DeleteTemporaryDirectory(temporarySymbolsDirectory); @@ -446,21 +453,27 @@ await PublishSymbolsHelper.PublishAsync( var serverPath = server.Key; var token = server.Value; symbolLog.AppendLine($"Publishing pdbFiles to {serverPath}:"); - - await PublishSymbolsHelper.PublishAsync( - Log, - serverPath, - token, - symbolFiles, - filesToSymbolServer, - null, - ExpirationInDays, - false, - publishSpecialClrFiles, - null, - false, - false, - true); + try + { + await PublishSymbolsHelper.PublishAsync( + Log, + serverPath, + token, + symbolFiles, + filesToSymbolServer, + null, + ExpirationInDays, + false, + publishSpecialClrFiles, + null, + false, + false, + true); + } + catch(Exception ex) + { + Log.LogError(ex.Message); + } } Log.LogMessage(MessageImportance.High, $"Successfully published pdb files"); } @@ -471,7 +484,7 @@ await PublishSymbolsHelper.PublishAsync( } /// - /// Publishes symbol, dll and pdb files to symbol server. + /// Decides how to publish the symbol, dll and pdb files /// /// Path to dll and pdb files /// Token to authenticate msdl @@ -489,38 +502,68 @@ public async Task HandleSymbolPublishingAsync ( Dictionary> buildAssets, string temporarySymbolsLocation = null) { - if (!UseApiOverride) + if (!UseStreamingPublishing) + { + await PublishSymbolsfromBlobArtifactsAsync(pdbArtifactsBasePath,msdlToken, + symWebToken,symbolPublishingExclusionsFile,publishSpecialClrFiles,temporarySymbolsLocation); + } + else + { + await PublishSymbolsUsingStreamingAsync(pdbArtifactsBasePath, msdlToken, + symWebToken, symbolPublishingExclusionsFile, publishSpecialClrFiles, buildAssets); + } + } + + /// + /// Publishes symbol, dll and pdb files to symbol server. + /// + /// Path to dll and pdb files + /// Token to authenticate msdl + /// Token to authenticate symweb + /// Right now we do not add any files to this, so this is going to be null + /// Path to Symbol.nupkgs + /// If true, the special coreclr module indexed files like DBI, DAC and SOS are published + /// + public async Task PublishSymbolsfromBlobArtifactsAsync( + string pdbArtifactsBasePath, + string msdlToken, + string symWebToken, + string symbolPublishingExclusionsFile, + bool publishSpecialClrFiles, + string temporarySymbolsLocation = null) + { + if (Directory.Exists(temporarySymbolsLocation)) { StringBuilder symbolLog = new StringBuilder(); symbolLog.AppendLine("Publishing Symbols to Symbol server: "); - if (Directory.Exists(temporarySymbolsLocation)) - { - string[] fileEntries = Directory.GetFiles(temporarySymbolsLocation); + string[] fileEntries = Directory.GetFiles(temporarySymbolsLocation); - var category = TargetFeedContentType.Symbols; + var category = TargetFeedContentType.Symbols; - HashSet feedConfigsForSymbols = FeedConfigs[category]; + HashSet feedConfigsForSymbols = FeedConfigs[category]; - Dictionary serversToPublish = - GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); + Dictionary serversToPublish = + GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); - IEnumerable filesToSymbolServer = null; - if (Directory.Exists(pdbArtifactsBasePath)) - { - var pdbEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.pdb", - System.IO.SearchOption.AllDirectories); - var dllEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.dll", - System.IO.SearchOption.AllDirectories); - filesToSymbolServer = pdbEntries.Concat(dllEntries); - } + IEnumerable filesToSymbolServer = null; + if (Directory.Exists(pdbArtifactsBasePath)) + { + var pdbEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.pdb", + System.IO.SearchOption.AllDirectories); + var dllEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.dll", + System.IO.SearchOption.AllDirectories); + filesToSymbolServer = pdbEntries.Concat(dllEntries); + } - foreach (var server in serversToPublish) + foreach (var server in serversToPublish) + { + var serverPath = server.Key; + var token = server.Value; + symbolLog.AppendLine($"Publishing symbol packages to {serverPath}:"); + symbolLog.AppendLine( + $"Performing symbol publishing...\nSymbolServerPath : ${serverPath} \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false \nTotal number of symbol files : {fileEntries.Length} "); + try { - var serverPath = server.Key; - var token = server.Value; - symbolLog.AppendLine($"Publishing symbol packages to {serverPath}:"); - symbolLog.AppendLine( - $"Performing symbol publishing...\nSymbolServerPath : ${serverPath} \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false \nTotal number of symbol files : {fileEntries.Length} "); await PublishSymbolsHelper.PublishAsync( Log, serverPath, @@ -535,21 +578,21 @@ await PublishSymbolsHelper.PublishAsync( false, false, true); - symbolLog.AppendLine("Successfully published to Symbol Server."); - symbolLog.AppendLine(); - Log.LogMessage(MessageImportance.High, symbolLog.ToString()); - symbolLog.Clear(); } - } - else - { - Log.LogError("Temporary symbols directory does not exists."); + catch (Exception ex) + { + Log.LogError(ex.Message); + } + + symbolLog.AppendLine("Successfully published to Symbol Server."); + symbolLog.AppendLine(); + Log.LogMessage(MessageImportance.High, symbolLog.ToString()); + symbolLog.Clear(); } } else { - await HandleSymbolPublishingOneByOneAsync(pdbArtifactsBasePath, msdlToken, - symWebToken, symbolPublishingExclusionsFile, publishSpecialClrFiles, buildAssets); + Log.LogError("Temporary symbols directory does not exists."); } } @@ -606,6 +649,7 @@ protected async Task HandlePackagePublishingAsync(Dictionary FilterPackages(HashSet + /// Creates Azdo client + /// + /// Name of the org + /// If client is used for downloading the artifact then AutomaticDecompression has to be set, else it is not required + /// If it is internal or public + /// For downloading file we use Version 4.1-preview.4 for other requests we use 6.0 + /// + public HttpClient CreateAzdoClient(string accountName, bool download, string projectName = null, string versionOverride = null) { HttpClientHandler handler = new HttpClientHandler { CheckCertificateRevocationList = true } ; if (download) { handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; } - string address = $"https://{baseAddressSubpath}dev.azure.com/{accountName}/"; + string address = $"https://dev.azure.com/{accountName}/"; if (!string.IsNullOrEmpty(projectName)) { address += $"{projectName}/"; @@ -715,18 +768,18 @@ private async Task GetContainerIdAsync() /// /// Download artifact file using Azure API /// - /// - /// - /// - /// - /// + /// Azdo client + /// If it is PackageArtifacts or BlobArtifacts + /// ContainerId where the packageArtifact and BlobArtifacts are stored + /// Name the file we are trying to download + /// Path where the file is being downloaded /// private async Task DownloadFileAsync(HttpClient client, string artifact, string containerId, string fileName, string path) { string uri = $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/_apis/resources/Containers/{containerId}?itemPath=/{artifact}/{fileName}&isShallow=true&api-version={AzureApiVersionForFileDownload}"; - Log.LogMessage($"download file uri = {uri}"); + Log.LogMessage(MessageImportance.Low, $"download file uri = {uri}"); Exception mostRecentlyCaughtException = null; bool success = await RetryHandler.RunAsync(async attempt => { @@ -762,7 +815,7 @@ private async Task DownloadFileAsync(HttpClient client, string artifact, string }).ConfigureAwait(false); if (!success) { - throw new Exception($"Failed to download local file '{path}' after {RetryHandler.MaxAttempts} attempts. See inner exception for details."); + throw new Exception($"Failed to download local file '{path}' after {RetryHandler.MaxAttempts} attempts. See inner exception for details, {mostRecentlyCaughtException }"); } } @@ -792,7 +845,7 @@ protected async Task HandleBlobPublishingAsync(Dictionary Log.LogMessage(MessageImportance.High, $"Blob {blob.Id} ({shippingString}) should go to {feedConfig.TargetURL} ({isolatedString}{internalString})"); } - if (!UseApiOverride) + if (!UseStreamingPublishing) { switch (feedConfig.Type) { @@ -907,58 +960,74 @@ public void SplitArtifactsInCategories(BuildModel buildModel) } } - private async Task PublishPackagesToAzDoNugetFeedAsync( + private async Task PublishPackagesFromPackageArtifactsToAzDoNugetFeedAsync( HashSet packagesToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig) { - if (!UseApiOverride) - { - await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, - async (feed, httpClient, package, feedAccount, feedVisibility, feedName) => - { - string localPackagePath = - Path.Combine(PackageAssetsBasePath, $"{package.Id}.{package.Version}.nupkg"); - if (!File.Exists(localPackagePath)) - { - Log.LogError($"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); - return; - } - - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.NugetFeed); - - await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, package.Version, - feedAccount, feedVisibility, feedName); - }); - } - else - { - string containerId = await GetContainerIdAsync(); - string localPackagePath = ""; - using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true); - foreach (var package in packagesToPublish) + await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, + async (feed, httpClient, package, feedAccount, feedVisibility, feedName) => { - var packageFilename = $"{package.Id}.{package.Version}.nupkg"; - string temporaryPackageDirectory = - Path.GetFullPath(Path.Combine(ArtifactsBasePath,Guid.NewGuid().ToString())); - EnsureTemporaryDirectoryExists(temporaryPackageDirectory); - localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - await DownloadFileAsync(client, "PackageArtifacts", containerId, - packageFilename, - localPackagePath); + string localPackagePath = + Path.Combine(PackageAssetsBasePath, $"{package.Id}.{package.Version}.nupkg"); if (!File.Exists(localPackagePath)) { - Log.LogError( - $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + Log.LogError($"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); return; } + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); - await PushPackageToNugetFeed(feedConfig, localPackagePath, package.Id, package.Version); - File.Delete(localPackagePath); - Directory.Delete(temporaryPackageDirectory); + + await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, package.Version, + feedAccount, feedVisibility, feedName); + }); + } + + private async Task PublishPackagesUsingStreamingToAzdoNugetAsync( + HashSet packagesToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + string containerId = await GetContainerIdAsync(); + string localPackagePath = ""; + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true); + foreach (var package in packagesToPublish) + { + var packageFilename = $"{package.Id}.{package.Version}.nupkg"; + string temporaryPackageDirectory = + Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryPackageDirectory); + localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); + await DownloadFileAsync(client, "PackageArtifacts", containerId, + packageFilename, + localPackagePath); + if (!File.Exists(localPackagePath)) + { + Log.LogError( + $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; } + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); + await PushPackageToNugetFeed(feedConfig, localPackagePath, package.Id, package.Version); + File.Delete(localPackagePath); + Directory.Delete(temporaryPackageDirectory); + } + } + + private async Task PublishPackagesToAzDoNugetFeedAsync( + HashSet packagesToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + if (!UseStreamingPublishing) + { + await PublishPackagesFromPackageArtifactsToAzDoNugetFeedAsync(packagesToPublish, buildAssets, feedConfig); + } + else + { + await PublishPackagesUsingStreamingToAzdoNugetAsync(packagesToPublish, buildAssets, feedConfig); } } @@ -1145,74 +1214,90 @@ private async Task PublishBlobsToAzDoNugetFeedAsync( } else { - Log.LogWarning($"AzDO feed publishing not available for blobs. Blob '{blob.Id}' was not published."); + Log.LogWarning( + $"AzDO feed publishing not available for blobs. Blob '{blob.Id}' was not published."); } } - if (!UseApiOverride) + if (!UseStreamingPublishing) { - await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, - async (feed, httpClient, blob, feedAccount, feedVisibility, feedName) => - { - if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.Container)) - { - // Determine the local path to the blob - string fileName = Path.GetFileName(blob.Id); - string localBlobPath = Path.Combine(BlobAssetsBasePath, fileName); - if (!File.Exists(localBlobPath)) - { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); - return; - } - - string id; - string version; - // Determine package ID and version by asking the nuget libraries - using var packageReader = new PackageArchiveReader(localBlobPath); - - PackageIdentity packageIdentity = packageReader.GetIdentity(); - id = packageIdentity.Id; - version = packageIdentity.Version.ToString(); - await PushNugetPackageAsync(feed, httpClient, localBlobPath, id, version, feedAccount, - feedVisibility, feedName); - } - }); + await PublishBlobsFromBlobArtifactsToAzDoNugetAsync(packagesToPublish, buildAssets, feedConfig); } else { - string containerId = await GetContainerIdAsync(); - using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true, AzureProject); - foreach (var blob in blobsToPublish) + await PublishBlobsUsingStreamingToAzDoNugetAsync(packagesToPublish, buildAssets, feedConfig); + } + } + + private async Task PublishBlobsFromBlobArtifactsToAzDoNugetAsync( + HashSet blobsToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + await PushNugetPackagesAsync(blobsToPublish, feedConfig, maxClients: MaxClients, + async (feed, httpClient, blob, feedAccount, feedVisibility, feedName) => { if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.Container)) { - string temporaryBlobDirectory = CreateTemporaryDirectory(); + // Determine the local path to the blob string fileName = Path.GetFileName(blob.Id); - string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); - await DownloadFileAsync(client, "BlobArtifacts", containerId, - fileName, - localBlobPath); + string localBlobPath = Path.Combine(BlobAssetsBasePath, fileName); if (!File.Exists(localBlobPath)) { Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + return; } + string id; string version; + // Determine package ID and version by asking the nuget libraries using var packageReader = new PackageArchiveReader(localBlobPath); PackageIdentity packageIdentity = packageReader.GetIdentity(); id = packageIdentity.Id; version = packageIdentity.Version.ToString(); - await PushBlobToNugetFeed(feedConfig, localBlobPath, id, version); - DeleteTemporaryFile(localBlobPath); - DeleteTemporaryDirectory(temporaryBlobDirectory); + await PushNugetPackageAsync(feed, httpClient, localBlobPath, id, version, feedAccount, + feedVisibility, feedName); + } + }); + } + + private async Task PublishBlobsUsingStreamingToAzDoNugetAsync( + HashSet blobsToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + string containerId = await GetContainerIdAsync(); + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true, AzureProject); + foreach (var blob in blobsToPublish) + { + if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container)) + { + string temporaryBlobDirectory = CreateTemporaryDirectory(); + string fileName = Path.GetFileName(blob.Id); + string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); + await DownloadFileAsync(client, "BlobArtifacts", containerId, + fileName, + localBlobPath); + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); } + string id; + string version; + using var packageReader = new PackageArchiveReader(localBlobPath); + + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); + await PushBlobToNugetFeed(feedConfig, localBlobPath, id, version); + DeleteTemporaryFile(localBlobPath); + DeleteTemporaryDirectory(temporaryBlobDirectory); } } } - private async Task PushBlobToNugetFeed(TargetFeedConfig feedConfig, string localBlobPath, string id, string version) { @@ -1246,85 +1331,104 @@ await PushNugetPackageAsync(feedConfig, httpClient, localBlobPath, id, } - private async Task PublishPackagesToAzureStorageNugetFeedAsync( - HashSet packagesToPublish, + private async Task PublishPackagesToAzureStorageNugetByDownloadingEntireFolderAsync(HashSet packagesToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig) { - if (UseApiOverride) + var packages = packagesToPublish.Select(p => { - var packages = packagesToPublish.Select(p => - { - var localPackagePath = Path.Combine(PackageAssetsBasePath, $"{p.Id}.{p.Version}.nupkg"); - if (!File.Exists(localPackagePath)) - { - Log.LogError($"Could not locate '{p.Id}.{p.Version}' at '{localPackagePath}'"); - } - return localPackagePath; - }); - - if (Log.HasLoggedErrors) + var localPackagePath = Path.Combine(PackageAssetsBasePath, $"{p.Id}.{p.Version}.nupkg"); + if (!File.Exists(localPackagePath)) { - return; + Log.LogError($"Could not locate '{p.Id}.{p.Version}' at '{localPackagePath}'"); } + return localPackagePath; + }); - var blobFeedAction = CreateBlobFeedAction(feedConfig); + if (Log.HasLoggedErrors) + { + return; + } - var pushOptions = new PushOptions - { - AllowOverwrite = feedConfig.AllowOverwrite, - PassIfExistingItemIdentical = true - }; + var blobFeedAction = CreateBlobFeedAction(feedConfig); - packagesToPublish - .ToList() - .ForEach(package => TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.NugetFeed)); + var pushOptions = new PushOptions + { + AllowOverwrite = feedConfig.AllowOverwrite, + PassIfExistingItemIdentical = true + }; - await blobFeedAction.PushToFeedAsync(packages, pushOptions); - } - else + packagesToPublish + .ToList() + .ForEach(package => TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed)); + + await blobFeedAction.PushToFeedAsync(packages, pushOptions); + } + + private async Task PublishPackagesToAzureStorageUsingStreamingAsync( + HashSet packagesToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + string containerId = await GetContainerIdAsync(); + var blobFeedAction = CreateBlobFeedAction(feedConfig); + + var pushOptions = new PushOptions { - string containerId = await GetContainerIdAsync(); - var blobFeedAction = CreateBlobFeedAction(feedConfig); + AllowOverwrite = feedConfig.AllowOverwrite, + PassIfExistingItemIdentical = true + }; + string localPackagePath = ""; + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true); - var pushOptions = new PushOptions + foreach (var package in packagesToPublish) + { + string temporaryPackageDirectory = CreateTemporaryDirectory(); + var packageFilename = $"{package.Id}.{package.Version}.nupkg"; + localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); + await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, + localPackagePath); + if (!File.Exists(localPackagePath)) { - AllowOverwrite = feedConfig.AllowOverwrite, - PassIfExistingItemIdentical = true - }; - string localPackagePath = ""; - using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true); + Log.LogError( + $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; + } - foreach (var package in packagesToPublish) + TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); + IEnumerable items = new List(); + items.ToList().Add(localPackagePath); + Log.LogMessage(MessageImportance.Low, $"START pushing {packageFilename} to feed"); + try { - string temporaryPackageDirectory = CreateTemporaryDirectory(); - var packageFilename = $"{package.Id}.{package.Version}.nupkg"; - localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, - localPackagePath); - if (!File.Exists(localPackagePath)) - { - Log.LogError( - $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); - return; - } - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.NugetFeed); - IEnumerable items = new List(); - items.ToList().Add(localPackagePath); - Log.LogMessage(MessageImportance.Low, $"START pushing {packageFilename} to feed"); - try - { - await blobFeedAction.PushAsync(items, pushOptions); - } - catch (Exception e) - { - Log.LogErrorFromException(e); - } - DeleteTemporaryFile(localPackagePath); - DeleteTemporaryDirectory(temporaryPackageDirectory); + await blobFeedAction.PushAsync(items, pushOptions); } + catch (Exception e) + { + Log.LogErrorFromException(e); + } + + DeleteTemporaryFile(localPackagePath); + DeleteTemporaryDirectory(temporaryPackageDirectory); + } + } + + private async Task PublishPackagesToAzureStorageNugetFeedAsync( + HashSet packagesToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + if (!UseStreamingPublishing) + { + await PublishPackagesToAzureStorageNugetByDownloadingEntireFolderAsync(packagesToPublish, buildAssets, + feedConfig); + } + else + { + await PublishPackagesToAzureStorageUsingStreamingAsync(packagesToPublish, buildAssets, + feedConfig); } } @@ -1345,89 +1449,107 @@ private async Task PublishBlobsToAzureStorageNugetFeedAsync( Dictionary> buildAssets, TargetFeedConfig feedConfig) { - if (!UseApiOverride) + if (!UseStreamingPublishing) { - var blobs = blobsToPublish - .Select(blob => - { - var fileName = Path.GetFileName(blob.Id); - var localBlobPath = Path.Combine(BlobAssetsBasePath, fileName); - if (!File.Exists(localBlobPath)) - { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); - } - - return new Microsoft.Build.Utilities.TaskItem(localBlobPath, new Dictionary - { - {"RelativeBlobPath", blob.Id} - }); - }) - .ToArray(); + await PublishBlobsToAzureStorageNugetAsync(blobsToPublish, buildAssets, feedConfig); + } + else + { + await PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync(blobsToPublish, buildAssets, + feedConfig); + } - if (Log.HasLoggedErrors) - { - return; - } + if (LinkManager == null) + { + LinkManager = new LatestLinksManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, AkaMSGroupOwner, + AkaMSCreatedBy, AkaMsOwners, Log); + } - var blobFeedAction = CreateBlobFeedAction(feedConfig); - var pushOptions = new PushOptions - { - AllowOverwrite = feedConfig.AllowOverwrite, - PassIfExistingItemIdentical = true - }; + // The latest links should be updated only after the publishing is complete, to avoid + // dead links in the interim. + await LinkManager.CreateOrUpdateLatestLinksAsync(blobsToPublish, feedConfig, + PublishingConstants.ExpectedFeedUrlSuffix.Length); + } - blobsToPublish - .ToList() - .ForEach(blob => TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.Container)); + private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync( + HashSet blobsToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + string containerId = await GetContainerIdAsync(); + var blobFeedAction = CreateBlobFeedAction(feedConfig); + var pushOptions = new PushOptions + { + AllowOverwrite = feedConfig.AllowOverwrite, + PassIfExistingItemIdentical = true + }; + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true, AzureProject); - await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients: MaxClients, pushOptions); - } - else + foreach (var blob in blobsToPublish) { - string containerId = await GetContainerIdAsync(); - var blobFeedAction = CreateBlobFeedAction(feedConfig); - var pushOptions = new PushOptions + string temporaryBlobDirectory = CreateTemporaryDirectory(); + var fileName = Path.GetFileName(blob.Id); + var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); + await DownloadFileAsync(client, "BlobArtifacts", containerId, fileName, + localBlobPath); + if (!File.Exists(localBlobPath)) { - AllowOverwrite = feedConfig.AllowOverwrite, - PassIfExistingItemIdentical = true - }; - using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true, AzureProject); + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + } + + var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, + new Dictionary + { + {"RelativeBlobPath", blob.Id} + }); + TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container); + await blobFeedAction.UploadAssetAsync(item, pushOptions); + DeleteTemporaryFile(localBlobPath); + DeleteTemporaryDirectory(temporaryBlobDirectory); + } + } - foreach (var blob in blobsToPublish) + private async Task PublishBlobsToAzureStorageNugetAsync( + HashSet blobsToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + var blobs = blobsToPublish + .Select(blob => { - string temporaryBlobDirectory = CreateTemporaryDirectory(); - //Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); - //EnsureTemporaryDirectoryExists(temporaryBlobDirectory); var fileName = Path.GetFileName(blob.Id); - var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); - await DownloadFileAsync(client, "BlobArtifacts", containerId, fileName, - localBlobPath); + var localBlobPath = Path.Combine(BlobAssetsBasePath, fileName); if (!File.Exists(localBlobPath)) { Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); } - var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, - new Dictionary - { - {"RelativeBlobPath", blob.Id} - }); - TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.Container); - await blobFeedAction.UploadAssetAsync(item, pushOptions); - DeleteTemporaryFile(localBlobPath); - DeleteTemporaryDirectory(temporaryBlobDirectory); - } - } - if (LinkManager == null) + return new Microsoft.Build.Utilities.TaskItem(localBlobPath, new Dictionary + { + {"RelativeBlobPath", blob.Id} + }); + }) + .ToArray(); + + if (Log.HasLoggedErrors) { - LinkManager = new LatestLinksManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, AkaMSGroupOwner, AkaMSCreatedBy, AkaMsOwners, Log); + return; } - // The latest links should be updated only after the publishing is complete, to avoid - // dead links in the interim. - await LinkManager.CreateOrUpdateLatestLinksAsync(blobsToPublish, feedConfig, PublishingConstants.ExpectedFeedUrlSuffix.Length); + var blobFeedAction = CreateBlobFeedAction(feedConfig); + var pushOptions = new PushOptions + { + AllowOverwrite = feedConfig.AllowOverwrite, + PassIfExistingItemIdentical = true + }; + + blobsToPublish + .ToList() + .ForEach(blob => TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + AddAssetLocationToAssetAssetLocationType.Container)); + + await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients: MaxClients, pushOptions); } private BlobFeedAction CreateBlobFeedAction(TargetFeedConfig feedConfig) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index 6ee2b38df8f..37c9a534b05 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -1,16 +1,16 @@ - // Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.Build.Framework; -using Microsoft.DotNet.Build.Tasks.Feed.Model; -using Microsoft.DotNet.Maestro.Client; -using Microsoft.DotNet.Maestro.Client.Models; -using Microsoft.DotNet.VersionTools.BuildManifest.Model; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Microsoft.DotNet.Build.Tasks.Feed.Model; +using Microsoft.DotNet.Maestro.Client; +using Microsoft.DotNet.Maestro.Client.Models; +using Microsoft.DotNet.VersionTools.BuildManifest.Model; namespace Microsoft.DotNet.Build.Tasks.Feed { @@ -190,7 +190,7 @@ public override async Task ExecuteAsync() } string temporarySymbolsLocation =""; - if (!UseApiOverride) + if (!UseStreamingPublishing) { temporarySymbolsLocation = From 5db58cc21632a63f6795c1fe22dae8dc152a202e Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Tue, 30 Mar 2021 18:55:29 -0700 Subject: [PATCH 20/32] more comments addressed --- .../SdkTasks/PublishArtifactsInManifest.proj | 2 + .../src/BlobFeedAction.cs | 78 +++---------------- .../src/PublishArtifactsInManifestBase.cs | 25 ++++-- .../src/PublishArtifactsInManifestV3.cs | 8 +- 4 files changed, 35 insertions(+), 78 deletions(-) diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index aea0ae309a6..746bc65f582 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -88,6 +88,7 @@ + @@ -107,6 +108,7 @@ $(ChecksumsAzureAccountKey) $(InternalChecksumsAzureAccountKey) false + false PushItemsToFeedAsync( return !Log.HasLoggedErrors; } - public async Task PublishToFlatContainerAsync(IEnumerable taskItems, int maxClients, PushOptions pushOptions) + public async Task PublishToFlatContainerAsync(IEnumerable taskItems, int maxClients, + PushOptions pushOptions) { if (taskItems.Any()) { using (var clientThrottle = new SemaphoreSlim(maxClients, maxClients)) { await System.Threading.Tasks.Task.WhenAll(taskItems.Select( - item => - { - return UploadAssetAsync(item, clientThrottle, pushOptions); - } + item => { return UploadAssetAsync(item, pushOptions, clientThrottle); } )); } } @@ -140,7 +138,8 @@ await System.Threading.Tasks.Task.WhenAll(taskItems.Select( public async Task UploadAssetAsync( ITaskItem item, - PushOptions options) + PushOptions options, + SemaphoreSlim clientThrottle = null) { string relativeBlobPath = item.GetMetadata("RelativeBlobPath"); @@ -162,70 +161,12 @@ public async Task UploadAssetAsync( } Log.LogMessage($"Uploading {relativeBlobPath}"); - try - { - AzureStorageUtils blobUtils = new AzureStorageUtils(AccountName, AccountKey, ContainerName); - if (!options.AllowOverwrite && await blobUtils.CheckIfBlobExistsAsync(relativeBlobPath)) - { - if (options.PassIfExistingItemIdentical) - { - if (!await blobUtils.IsFileIdenticalToBlobAsync(item.ItemSpec, relativeBlobPath)) - { - Log.LogError( - $"Item '{item}' already exists with different contents " + - $"at '{relativeBlobPath}'"); - } - } - else - { - Log.LogError($"Item '{item}' already exists at '{relativeBlobPath}'"); - } - } - else - { - using (FileStream stream = - new FileStream(item.ItemSpec, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - Log.LogMessage($"Uploading {item} to {relativeBlobPath}."); - await blobUtils.UploadBlockBlobAsync(item.ItemSpec, relativeBlobPath, stream); - } - } - } - catch (Exception exc) - { - Log.LogError($"Unable to upload to {relativeBlobPath} in Azure Storage account {AccountName}/{ContainerName} due to {exc}."); - throw; - } - } - public async Task UploadAssetAsync( - ITaskItem item, - SemaphoreSlim clientThrottle, - PushOptions options) - { - string relativeBlobPath = item.GetMetadata("RelativeBlobPath"); - - if (string.IsNullOrEmpty(relativeBlobPath)) + if (clientThrottle != null) { - string fileName = Path.GetFileName(item.ItemSpec); - string recursiveDir = item.GetMetadata("RecursiveDir"); - relativeBlobPath = $"{recursiveDir}{fileName}"; + await clientThrottle.WaitAsync(); } - relativeBlobPath = $"{RelativePath}{relativeBlobPath}".Replace("\\", "/"); - - if (relativeBlobPath.Contains("//")) - { - Log.LogError( - $"Item '{item.ItemSpec}' RelativeBlobPath contains virtual directory " + - $"without name (double forward slash): '{relativeBlobPath}'"); - return; - } - - Log.LogMessage($"Uploading {relativeBlobPath}"); - - await clientThrottle.WaitAsync(); - try { AzureStorageUtils blobUtils = new AzureStorageUtils(AccountName, AccountKey, ContainerName); @@ -263,7 +204,10 @@ public async Task UploadAssetAsync( } finally { - clientThrottle.Release(); + if (clientThrottle != null) + { + clientThrottle.Release(); + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index eb65e95ea4f..481ff3cdba9 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -170,6 +170,10 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities DelayBase = 2.5 // 2.5 ^ 5 = ~1.5 minutes max wait between retries }; + private const string BlobArtifacts = "BlobArtifacts"; + + private const string PackageArtifacts = "PackageArtifacts"; + public override bool Execute() { return ExecuteAsync().GetAwaiter().GetResult(); @@ -392,7 +396,7 @@ public async Task PublishSymbolsUsingStreamingAsync( { string temporarySymbolsDirectory = CreateTemporaryDirectory(); localSymbolPath = Path.Combine(temporarySymbolsDirectory, symbol); - await DownloadFileAsync(client, "BlobArtifacts", containerId, symbol, localSymbolPath); + await DownloadFileAsync(client, BlobArtifacts, containerId, symbol, localSymbolPath); Log.LogMessage(MessageImportance.Low, $"Local Symbol path to downloaded file {localSymbolPath}"); List symbolFiles = new List(); @@ -744,7 +748,7 @@ private async Task GetContainerIdAsync() foreach (var artifact in buildArtifacts.value) { - if (string.Equals(artifact.name, "BlobArtifacts")) + if (string.Equals(artifact.name, BlobArtifacts)) { string[] segment = artifact.resource.data.Split('/'); containerId = segment[1]; @@ -999,7 +1003,7 @@ private async Task PublishPackagesUsingStreamingToAzdoNugetAsync( Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); EnsureTemporaryDirectoryExists(temporaryPackageDirectory); localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - await DownloadFileAsync(client, "PackageArtifacts", containerId, + await DownloadFileAsync(client, PackageArtifacts, containerId, packageFilename, localPackagePath); if (!File.Exists(localPackagePath)) @@ -1387,7 +1391,7 @@ private async Task PublishPackagesToAzureStorageUsingStreamingAsync( string temporaryPackageDirectory = CreateTemporaryDirectory(); var packageFilename = $"{package.Id}.{package.Version}.nupkg"; localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - await DownloadFileAsync(client, "PackageArtifacts", containerId, packageFilename, + await DownloadFileAsync(client, PackageArtifacts, containerId, packageFilename, localPackagePath); if (!File.Exists(localPackagePath)) { @@ -1504,7 +1508,7 @@ await DownloadFileAsync(client, "BlobArtifacts", containerId, fileName, }); TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.Container); - await blobFeedAction.UploadAssetAsync(item, pushOptions); + await blobFeedAction.UploadAssetAsync(item, pushOptions, null); DeleteTemporaryFile(localBlobPath); DeleteTemporaryDirectory(temporaryBlobDirectory); } @@ -1657,7 +1661,7 @@ public void DeleteTemporaryFiles(string temporaryLocation) /// /// Delete the files after publishing, this is part of cleanup /// - /// + /// public void DeleteTemporaryFile(string filePath) { try @@ -1682,7 +1686,14 @@ public void DeleteTemporaryDirectory(string temporaryLocation) { if (Directory.Exists(temporaryLocation)) { - Directory.Delete(temporaryLocation); + try + { + Directory.Delete(temporaryLocation); + } + catch (Exception ex) + { + Log.LogWarning(ex.Message); + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index 37c9a534b05..cb18fc0cd1e 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -189,7 +189,7 @@ public override async Task ExecuteAsync() return false; } - string temporarySymbolsLocation =""; + string temporarySymbolsLocation = ""; if (!UseStreamingPublishing) { @@ -199,8 +199,8 @@ public override async Task ExecuteAsync() EnsureTemporaryDirectoryExists(temporarySymbolsLocation); DeleteTemporaryFiles(temporarySymbolsLocation); - //Copying symbol files to temporary location is required because the symUploader API needs read/write access to the files, - //since we publish blobs and symbols in parallel this will cause IO exceptions. + // Copying symbol files to temporary location is required because the symUploader API needs read/write access to the files, + // since we publish blobs and symbols in parallel this will cause IO exceptions. CopySymbolFilesToTemporaryLocation(BuildModel, temporarySymbolsLocation); } @@ -208,7 +208,7 @@ await Task.WhenAll(new Task[] { HandlePackagePublishingAsync(buildAssets), HandleBlobPublishingAsync(buildAssets), HandleSymbolPublishingAsync(PdbArtifactsBasePath, MsdlToken, - SymWebToken, SymbolPublishingExclusionsFile, PublishSpecialClrFiles, buildAssets ,temporarySymbolsLocation) + SymWebToken, SymbolPublishingExclusionsFile, PublishSpecialClrFiles, buildAssets, temporarySymbolsLocation) }); DeleteTemporaryFiles(temporarySymbolsLocation); From 53f99f8731fd5705c3da3b6d23fc720b7ee25adf Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Tue, 30 Mar 2021 22:12:00 -0700 Subject: [PATCH 21/32] Update comment --- .../src/PublishArtifactsInManifestBase.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 481ff3cdba9..856cc92ae3a 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -730,6 +730,7 @@ public HttpClient CreateAzdoClient(string accountName, bool download, string pr /// /// Gets the container Id, that is going to be used in another API call to download the assets + /// ContainerId is the same for PackageArtifacts and BlobArtifacts /// /// private async Task GetContainerIdAsync() From 32cc9b06d1d0fe7ab69287e2107f8357fb136178 Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Wed, 31 Mar 2021 10:26:59 -0700 Subject: [PATCH 22/32] minor fix --- .../src/BlobFeedAction.cs | 84 ++++++++++--------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs index 410648bba89..791b440ece5 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs @@ -150,64 +150,72 @@ public async Task UploadAssetAsync( relativeBlobPath = $"{recursiveDir}{fileName}"; } - relativeBlobPath = $"{RelativePath}{relativeBlobPath}".Replace("\\", "/"); - - if (relativeBlobPath.Contains("//")) + if (!string.IsNullOrEmpty(relativeBlobPath)) { - Log.LogError( - $"Item '{item.ItemSpec}' RelativeBlobPath contains virtual directory " + - $"without name (double forward slash): '{relativeBlobPath}'"); - return; - } + relativeBlobPath = $"{RelativePath}{relativeBlobPath}".Replace("\\", "/"); - Log.LogMessage($"Uploading {relativeBlobPath}"); + if (relativeBlobPath.StartsWith("//")) + { + Log.LogError( + $"Item '{item.ItemSpec}' RelativeBlobPath contains virtual directory " + + $"without name (double forward slash): '{relativeBlobPath}'"); + return; + } - if (clientThrottle != null) - { - await clientThrottle.WaitAsync(); - } + Log.LogMessage($"Uploading {relativeBlobPath}"); - try - { - AzureStorageUtils blobUtils = new AzureStorageUtils(AccountName, AccountKey, ContainerName); + if (clientThrottle != null) + { + await clientThrottle.WaitAsync(); + } - if (!options.AllowOverwrite && await blobUtils.CheckIfBlobExistsAsync(relativeBlobPath)) + try { - if (options.PassIfExistingItemIdentical) + AzureStorageUtils blobUtils = new AzureStorageUtils(AccountName, AccountKey, ContainerName); + + if (!options.AllowOverwrite && await blobUtils.CheckIfBlobExistsAsync(relativeBlobPath)) { - if (!await blobUtils.IsFileIdenticalToBlobAsync(item.ItemSpec, relativeBlobPath)) + if (options.PassIfExistingItemIdentical) { - Log.LogError( - $"Item '{item}' already exists with different contents " + - $"at '{relativeBlobPath}'"); + if (!await blobUtils.IsFileIdenticalToBlobAsync(item.ItemSpec, relativeBlobPath)) + { + Log.LogError( + $"Item '{item}' already exists with different contents " + + $"at '{relativeBlobPath}'"); + } + } + else + { + Log.LogError($"Item '{item}' already exists at '{relativeBlobPath}'"); } } else { - Log.LogError($"Item '{item}' already exists at '{relativeBlobPath}'"); + using (FileStream stream = + new FileStream(item.ItemSpec, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + Log.LogMessage($"Uploading {item} to {relativeBlobPath}."); + await blobUtils.UploadBlockBlobAsync(item.ItemSpec, relativeBlobPath, stream); + } } } - else + catch (Exception exc) + { + Log.LogError( + $"Unable to upload to {relativeBlobPath} in Azure Storage account {AccountName}/{ContainerName} due to {exc}."); + throw; + } + finally { - using (FileStream stream = - new FileStream(item.ItemSpec, FileMode.Open, FileAccess.Read, FileShare.Read)) + if (clientThrottle != null) { - Log.LogMessage($"Uploading {item} to {relativeBlobPath}."); - await blobUtils.UploadBlockBlobAsync(item.ItemSpec, relativeBlobPath, stream); + clientThrottle.Release(); } } } - catch (Exception exc) - { - Log.LogError($"Unable to upload to {relativeBlobPath} in Azure Storage account {AccountName}/{ContainerName} due to {exc}."); - throw; - } - finally + else { - if (clientThrottle != null) - { - clientThrottle.Release(); - } + Log.LogError($"Relative blob path is empty."); } } From 05ba152d2ba94fe3241e48de38e64b3561f0a65e Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 1 Apr 2021 01:07:15 -0700 Subject: [PATCH 23/32] moooore comments --- .../src/PublishArtifactsInManifestBase.cs | 430 +++++++++++++----- 1 file changed, 309 insertions(+), 121 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 856cc92ae3a..d4c81b95724 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -125,8 +125,6 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities public bool UseStreamingPublishing { get; set; } - public int Retries = 5; - public readonly Dictionary> FeedConfigs = new Dictionary>(); @@ -164,16 +162,17 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities /// public int RetryDelayMilliseconds { get; set; } = 5000; - public ExponentialRetry RetryHandler = new ExponentialRetry + public readonly ExponentialRetry RetryHandler = new ExponentialRetry { MaxAttempts = 5, DelayBase = 2.5 // 2.5 ^ 5 = ~1.5 minutes max wait between retries }; - private const string BlobArtifacts = "BlobArtifacts"; - - private const string PackageArtifacts = "PackageArtifacts"; - + private enum ArtifactName + { + PackageArtifacts, + BlobArtifacts + } public override bool Execute() { return ExecuteAsync().GetAwaiter().GetResult(); @@ -361,8 +360,8 @@ public void CheckForStableAssetsInNonIsolatedFeeds() /// /// /// - /// - public async Task PublishSymbolsUsingStreamingAsync( + /// Task + public async Task PublishSymbolsUsingStreamingAsync( string pdbArtifactsBasePath, string msdlToken, string symWebToken, @@ -373,8 +372,14 @@ public async Task PublishSymbolsUsingStreamingAsync( StringBuilder symbolLog = new StringBuilder(); symbolLog.AppendLine("Publishing Symbols to Symbol server: "); var symbolCategory = TargetFeedContentType.Symbols; - string containerId = await GetContainerIdAsync(); + string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts); + + if (Log.HasLoggedErrors) + { + return; + } HashSet symbolsToPublish = new HashSet(); + //Get all the symbol file names foreach (var asset in buildAssets) { @@ -384,6 +389,7 @@ public async Task PublishSymbolsUsingStreamingAsync( symbolsToPublish.Add(Path.GetFileName(name)); } } + HashSet feedConfigsForSymbols = FeedConfigs[symbolCategory]; Dictionary serversToPublish = GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); @@ -392,20 +398,23 @@ public async Task PublishSymbolsUsingStreamingAsync( using (HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true)) { string localSymbolPath = ""; + foreach (var symbol in symbolsToPublish) { string temporarySymbolsDirectory = CreateTemporaryDirectory(); localSymbolPath = Path.Combine(temporarySymbolsDirectory, symbol); - await DownloadFileAsync(client, BlobArtifacts, containerId, symbol, localSymbolPath); - - Log.LogMessage(MessageImportance.Low, $"Local Symbol path to downloaded file {localSymbolPath}"); + + await DownloadFileAsync(client, ArtifactName.BlobArtifacts, containerId, symbol, localSymbolPath); + symbolLog.AppendLine($"Local Symbol path to downloaded file {localSymbolPath}"); List symbolFiles = new List(); symbolFiles.ToList().Add(localSymbolPath); + foreach (var server in serversToPublish) { var serverPath = server.Key; var token = server.Value; symbolLog.AppendLine($"Publishing symbol file {symbol} to {serverPath}:"); + try { await PublishSymbolsHelper.PublishAsync( @@ -428,17 +437,20 @@ await PublishSymbolsHelper.PublishAsync( Log.LogError(ex.Message); } } + DeleteTemporaryFile(localSymbolPath); DeleteTemporaryDirectory(temporarySymbolsDirectory); } + symbolLog.AppendLine( $"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false "); - symbolLog.Append($"\nTotal number of symbol files : {symbolsToPublish.Count}"); - symbolLog.AppendLine("\nSuccessfully published to Symbol Server."); + symbolLog.AppendLine($"Total number of symbol files : {symbolsToPublish.Count}"); + symbolLog.AppendLine("Successfully published to Symbol Server."); symbolLog.AppendLine(); Log.LogMessage(MessageImportance.High, symbolLog.ToString()); symbolLog.Clear(); } + // publishing pdb artifacts if (Directory.Exists(pdbArtifactsBasePath)) { @@ -451,7 +463,6 @@ await PublishSymbolsHelper.PublishAsync( if (filesToSymbolServer != null && filesToSymbolServer.Any()) { - List symbolFiles = null; foreach (var server in serversToPublish) { var serverPath = server.Key; @@ -463,7 +474,7 @@ await PublishSymbolsHelper.PublishAsync( Log, serverPath, token, - symbolFiles, + null, filesToSymbolServer, null, ExpirationInDays, @@ -474,13 +485,16 @@ await PublishSymbolsHelper.PublishAsync( false, true); } - catch(Exception ex) + + catch (Exception ex) { Log.LogError(ex.Message); } } + Log.LogMessage(MessageImportance.High, $"Successfully published pdb files"); } + else { Log.LogMessage(MessageImportance.Low, $"No pdb files to upload to symbol server."); @@ -496,7 +510,7 @@ await PublishSymbolsHelper.PublishAsync( /// Right now we do not add any files to this, so this is going to be null /// Path to Symbol.nupkgs /// If true, the special coreclr module indexed files like DBI, DAC and SOS are published - /// + /// Task public async Task HandleSymbolPublishingAsync ( string pdbArtifactsBasePath, string msdlToken, @@ -506,15 +520,26 @@ public async Task HandleSymbolPublishingAsync ( Dictionary> buildAssets, string temporarySymbolsLocation = null) { - if (!UseStreamingPublishing) + if (UseStreamingPublishing) { - await PublishSymbolsfromBlobArtifactsAsync(pdbArtifactsBasePath,msdlToken, - symWebToken,symbolPublishingExclusionsFile,publishSpecialClrFiles,temporarySymbolsLocation); + await PublishSymbolsUsingStreamingAsync( + pdbArtifactsBasePath, + msdlToken, + symWebToken, + symbolPublishingExclusionsFile, + publishSpecialClrFiles, + buildAssets); } + else { - await PublishSymbolsUsingStreamingAsync(pdbArtifactsBasePath, msdlToken, - symWebToken, symbolPublishingExclusionsFile, publishSpecialClrFiles, buildAssets); + await PublishSymbolsfromBlobArtifactsAsync( + pdbArtifactsBasePath, + msdlToken, + symWebToken, + symbolPublishingExclusionsFile, + publishSpecialClrFiles, + temporarySymbolsLocation); } } @@ -527,7 +552,7 @@ await PublishSymbolsUsingStreamingAsync(pdbArtifactsBasePath, msdlToken, /// Right now we do not add any files to this, so this is going to be null /// Path to Symbol.nupkgs /// If true, the special coreclr module indexed files like DBI, DAC and SOS are published - /// + /// Task public async Task PublishSymbolsfromBlobArtifactsAsync( string pdbArtifactsBasePath, string msdlToken, @@ -566,6 +591,7 @@ public async Task PublishSymbolsfromBlobArtifactsAsync( symbolLog.AppendLine($"Publishing symbol packages to {serverPath}:"); symbolLog.AppendLine( $"Performing symbol publishing...\nSymbolServerPath : ${serverPath} \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false \nTotal number of symbol files : {fileEntries.Length} "); + try { await PublishSymbolsHelper.PublishAsync( @@ -583,6 +609,7 @@ await PublishSymbolsHelper.PublishAsync( false, true); } + catch (Exception ex) { Log.LogError(ex.Message); @@ -594,6 +621,7 @@ await PublishSymbolsHelper.PublishAsync( symbolLog.Clear(); } } + else { Log.LogError("Temporary symbols directory does not exists."); @@ -606,7 +634,7 @@ await PublishSymbolsHelper.PublishAsync( /// /// /// - /// + /// A map of symbol server path => token to the symbol server public Dictionary GetTargetSymbolServers(HashSet feedConfigsForSymbols, string msdlToken, string symWebToken) { Dictionary serversToPublish = new Dictionary(); @@ -657,12 +685,16 @@ protected async Task HandlePackagePublishingAsync(Dictionary FilterPackages(HashSet /// Name of the org - /// If client is used for downloading the artifact then AutomaticDecompression has to be set, else it is not required - /// If it is internal or public - /// For downloading file we use Version 4.1-preview.4 for other requests we use 6.0 - /// - public HttpClient CreateAzdoClient(string accountName, bool download, string projectName = null, string versionOverride = null) + /// If client is used for downloading the artifact then AutomaticDecompression has to be set, else it is not required + /// Azure project name + /// Default version is 6.0 + /// HttpClient + public HttpClient CreateAzdoClient( + string accountName, + bool setAutomaticDecompression, + string projectName = null, + string versionOverride = null) { - HttpClientHandler handler = new HttpClientHandler { CheckCertificateRevocationList = true } ; - if (download) + HttpClientHandler handler = new HttpClientHandler {CheckCertificateRevocationList = true}; + if (setAutomaticDecompression) { handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; } + string address = $"https://dev.azure.com/{accountName}/"; if (!string.IsNullOrEmpty(projectName)) { address += $"{projectName}/"; } + var client = new HttpClient(handler) { BaseAddress = new Uri(address) @@ -732,8 +771,8 @@ public HttpClient CreateAzdoClient(string accountName, bool download, string pr /// Gets the container Id, that is going to be used in another API call to download the assets /// ContainerId is the same for PackageArtifacts and BlobArtifacts /// - /// - private async Task GetContainerIdAsync() + /// ContainerId + private async Task GetContainerIdAsync(ArtifactName artifactName) { string containerId = ""; using (HttpClient client = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject)) @@ -749,7 +788,7 @@ private async Task GetContainerIdAsync() foreach (var artifact in buildArtifacts.value) { - if (string.Equals(artifact.name, BlobArtifacts)) + if (string.Equals(artifact.name, artifactName)) { string[] segment = artifact.resource.data.Split('/'); containerId = segment[1]; @@ -765,6 +804,7 @@ private async Task GetContainerIdAsync() if (string.IsNullOrEmpty(containerId)) { Log.LogError("Container Id does not exists"); + } return containerId; } @@ -774,16 +814,19 @@ private async Task GetContainerIdAsync() /// Download artifact file using Azure API /// /// Azdo client - /// If it is PackageArtifacts or BlobArtifacts + /// If it is PackageArtifacts or BlobArtifacts /// ContainerId where the packageArtifact and BlobArtifacts are stored /// Name the file we are trying to download /// Path where the file is being downloaded - /// - private async Task DownloadFileAsync(HttpClient client, string artifact, string containerId, - string fileName, string path) + private async Task DownloadFileAsync( + HttpClient client, + ArtifactName artifactName, + string containerId, + string fileName, + string path) { string uri = - $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/_apis/resources/Containers/{containerId}?itemPath=/{artifact}/{fileName}&isShallow=true&api-version={AzureApiVersionForFileDownload}"; + $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/_apis/resources/Containers/{containerId}?itemPath=/{artifactName}/{fileName}&isShallow=true&api-version={AzureApiVersionForFileDownload}"; Log.LogMessage(MessageImportance.Low, $"download file uri = {uri}"); Exception mostRecentlyCaughtException = null; bool success = await RetryHandler.RunAsync(async attempt => @@ -792,13 +835,17 @@ private async Task DownloadFileAsync(HttpClient client, string artifact, string { CancellationTokenSource timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes)); + using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); using HttpResponseMessage response = await client.GetAsync(uri, timeoutTokenSource.Token); response.EnsureSuccessStatusCode(); - using var fs = new FileStream(path, FileMode.Create, - FileAccess.ReadWrite, FileShare.ReadWrite); + using var fs = new FileStream( + path, + FileMode.Create, + FileAccess.ReadWrite, + FileShare.ReadWrite); using var stream = await response.Content.ReadAsStreamAsync(); try @@ -810,6 +857,7 @@ private async Task DownloadFileAsync(HttpClient client, string artifact, string stream.Close(); fs.Close(); } + return true; } catch (HttpRequestException toStore) @@ -818,9 +866,11 @@ private async Task DownloadFileAsync(HttpClient client, string artifact, string return false; } }).ConfigureAwait(false); + if (!success) { - throw new Exception($"Failed to download local file '{path}' after {RetryHandler.MaxAttempts} attempts. See inner exception for details, {mostRecentlyCaughtException }"); + throw new Exception( + $"Failed to download local file '{path}' after {RetryHandler.MaxAttempts} attempts. See inner exception for details, {mostRecentlyCaughtException}"); } } @@ -847,35 +897,40 @@ protected async Task HandleBlobPublishingAsync(Dictionary string isolatedString = feedConfig.Isolated ? "Isolated" : "Non-Isolated"; string internalString = feedConfig.Internal ? $", Internal" : ", Public"; string shippingString = blob.NonShipping ? "NonShipping" : "Shipping"; - Log.LogMessage(MessageImportance.High, $"Blob {blob.Id} ({shippingString}) should go to {feedConfig.TargetURL} ({isolatedString}{internalString})"); + Log.LogMessage(MessageImportance.High, + $"Blob {blob.Id} ({shippingString}) should go to {feedConfig.TargetURL} ({isolatedString}{internalString})"); } - if (!UseStreamingPublishing) + switch (feedConfig.Type) { - switch (feedConfig.Type) - { - case FeedType.AzDoNugetFeed: - publishTasks.Add(PublishBlobsToAzDoNugetFeedAsync(filteredBlobs, buildAssets, + case FeedType.AzDoNugetFeed: + publishTasks.Add( + PublishBlobsToAzDoNugetFeedAsync( + filteredBlobs, + buildAssets, + feedConfig)); + break; + case FeedType.AzureStorageFeed: + publishTasks.Add( + PublishBlobsToAzureStorageNugetFeedAsync( + filteredBlobs, + buildAssets, feedConfig)); - break; - case FeedType.AzureStorageFeed: - publishTasks.Add( - PublishBlobsToAzureStorageNugetFeedAsync(filteredBlobs, buildAssets, - feedConfig)); - break; - default: - Log.LogError( - $"Unknown target feed type for category '{category}': '{feedConfig.Type}'."); - break; - } + break; + default: + Log.LogError( + $"Unknown target feed type for category '{category}': '{feedConfig.Type}'."); + break; } } } + else { Log.LogError($"No target feed configuration found for artifact category: '{category}'."); } } + await Task.WhenAll(publishTasks); } @@ -932,7 +987,7 @@ public void SplitArtifactsInCategories(BuildModel buildModel) } else { - PackagesByCategory[categoryKey] = new HashSet() { packageAsset }; + PackagesByCategory[categoryKey] = new HashSet() {packageAsset}; } } } @@ -959,7 +1014,7 @@ public void SplitArtifactsInCategories(BuildModel buildModel) } else { - BlobsByCategory[categoryKey] = new HashSet() { blobAsset }; + BlobsByCategory[categoryKey] = new HashSet() {blobAsset}; } } } @@ -981,11 +1036,21 @@ await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClien return; } - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + TryAddAssetLocation( + package.Id, + package.Version, + buildAssets, + feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); - await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, package.Version, - feedAccount, feedVisibility, feedName); + await PushNugetPackageAsync(feed, + httpClient, + localPackagePath, + package.Id, + package.Version, + feedAccount, + feedVisibility, + feedName); }); } @@ -994,9 +1059,16 @@ private async Task PublishPackagesUsingStreamingToAzdoNugetAsync( Dictionary> buildAssets, TargetFeedConfig feedConfig) { - string containerId = await GetContainerIdAsync(); + string containerId = await GetContainerIdAsync(ArtifactName.PackageArtifacts); + + if (Log.HasLoggedErrors) + { + return; + } string localPackagePath = ""; + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true); + foreach (var package in packagesToPublish) { var packageFilename = $"{package.Id}.{package.Version}.nupkg"; @@ -1004,18 +1076,41 @@ private async Task PublishPackagesUsingStreamingToAzdoNugetAsync( Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); EnsureTemporaryDirectoryExists(temporaryPackageDirectory); localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - await DownloadFileAsync(client, PackageArtifacts, containerId, + + await DownloadFileAsync( + client, + ArtifactName.PackageArtifacts, + containerId, packageFilename, localPackagePath); + if (!File.Exists(localPackagePath)) { Log.LogError( $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); return; } - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + + TryAddAssetLocation( + package.Id, + package.Version, + buildAssets, + feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); - await PushPackageToNugetFeed(feedConfig, localPackagePath, package.Id, package.Version); + + using HttpClient httpClient = new HttpClient(new HttpClientHandler + { + CheckCertificateRevocationList = true + }); + + httpClient.Timeout = TimeSpan.FromSeconds(180); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String( + Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); + + await PushPackageToNugetFeed(httpClient, feedConfig, localPackagePath, package.Id, package.Version); + File.Delete(localPackagePath); Directory.Delete(temporaryPackageDirectory); } @@ -1026,13 +1121,13 @@ private async Task PublishPackagesToAzDoNugetFeedAsync( Dictionary> buildAssets, TargetFeedConfig feedConfig) { - if (!UseStreamingPublishing) + if (UseStreamingPublishing) { - await PublishPackagesFromPackageArtifactsToAzDoNugetFeedAsync(packagesToPublish, buildAssets, feedConfig); + await PublishPackagesUsingStreamingToAzdoNugetAsync(packagesToPublish, buildAssets, feedConfig); } else { - await PublishPackagesUsingStreamingToAzdoNugetAsync(packagesToPublish, buildAssets, feedConfig); + await PublishPackagesFromPackageArtifactsToAzDoNugetFeedAsync(packagesToPublish, buildAssets, feedConfig); } } @@ -1041,7 +1136,10 @@ private async Task PublishPackagesToAzDoNugetFeedAsync( /// /// List of packages to publish /// Information about feed to publish to - public async Task PushNugetPackagesAsync(HashSet packagesToPublish, TargetFeedConfig feedConfig, int maxClients, + public async Task PushNugetPackagesAsync( + HashSet packagesToPublish, + TargetFeedConfig feedConfig, + int maxClients, Func packagePublishAction) { if (!packagesToPublish.Any()) @@ -1075,8 +1173,13 @@ await Task.WhenAll(packagesToPublish.Select(async packageToPublish => { // Wait to avoid starting too many processes. await clientThrottle.WaitAsync(); - await packagePublishAction(feedConfig, httpClient, packageToPublish, feedAccount, - feedVisibility, feedName); + await packagePublishAction( + feedConfig, + httpClient, + packageToPublish, + feedAccount, + feedVisibility, + feedName); } finally { @@ -1224,13 +1327,14 @@ private async Task PublishBlobsToAzDoNugetFeedAsync( } } - if (!UseStreamingPublishing) + if (UseStreamingPublishing) { - await PublishBlobsFromBlobArtifactsToAzDoNugetAsync(packagesToPublish, buildAssets, feedConfig); + await PublishBlobsUsingStreamingToAzDoNugetAsync(packagesToPublish, buildAssets, feedConfig); } + else { - await PublishBlobsUsingStreamingToAzDoNugetAsync(packagesToPublish, buildAssets, feedConfig); + await PublishBlobsFromBlobArtifactsToAzDoNugetAsync(packagesToPublish, buildAssets, feedConfig); } } @@ -1242,7 +1346,11 @@ private async Task PublishBlobsFromBlobArtifactsToAzDoNugetAsync( await PushNugetPackagesAsync(blobsToPublish, feedConfig, maxClients: MaxClients, async (feed, httpClient, blob, feedAccount, feedVisibility, feedName) => { - if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + if (TryAddAssetLocation( + blob.Id, + assetVersion: null, + buildAssets, + feedConfig, AddAssetLocationToAssetAssetLocationType.Container)) { // Determine the local path to the blob @@ -1262,8 +1370,16 @@ await PushNugetPackagesAsync(blobsToPublish, feedConfig, maxC PackageIdentity packageIdentity = packageReader.GetIdentity(); id = packageIdentity.Id; version = packageIdentity.Version.ToString(); - await PushNugetPackageAsync(feed, httpClient, localBlobPath, id, version, feedAccount, - feedVisibility, feedName); + + await PushNugetPackageAsync( + feed, + httpClient, + localBlobPath, + id, + version, + feedAccount, + feedVisibility, + feedName); } }); } @@ -1273,8 +1389,14 @@ private async Task PublishBlobsUsingStreamingToAzDoNugetAsync( Dictionary> buildAssets, TargetFeedConfig feedConfig) { - string containerId = await GetContainerIdAsync(); + string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts); + + if (Log.HasLoggedErrors) + { + return; + } using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true, AzureProject); + foreach (var blob in blobsToPublish) { if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, @@ -1283,9 +1405,11 @@ private async Task PublishBlobsUsingStreamingToAzDoNugetAsync( string temporaryBlobDirectory = CreateTemporaryDirectory(); string fileName = Path.GetFileName(blob.Id); string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); - await DownloadFileAsync(client, "BlobArtifacts", containerId, + + await DownloadFileAsync(client, ArtifactName.BlobArtifacts, containerId, fileName, localBlobPath); + if (!File.Exists(localBlobPath)) { Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); @@ -1297,12 +1421,19 @@ await DownloadFileAsync(client, "BlobArtifacts", containerId, PackageIdentity packageIdentity = packageReader.GetIdentity(); id = packageIdentity.Id; version = packageIdentity.Version.ToString(); - await PushBlobToNugetFeed(feedConfig, localBlobPath, id, version); + + await PushBlobToNugetFeed( + feedConfig, + localBlobPath, + id, + version); + DeleteTemporaryFile(localBlobPath); DeleteTemporaryDirectory(temporaryBlobDirectory); } } } + private async Task PushBlobToNugetFeed(TargetFeedConfig feedConfig, string localBlobPath, string id, string version) { @@ -1316,6 +1447,7 @@ private async Task PushBlobToNugetFeed(TargetFeedConfig feedConfig, string local string feedAccount = parsedUri.Groups["account"].Value; string feedVisibility = parsedUri.Groups["visibility"].Value; string feedName = parsedUri.Groups["feed"].Value; + using HttpClient httpClient = new HttpClient(new HttpClientHandler {CheckCertificateRevocationList = true}); httpClient.Timeout = TimeSpan.FromSeconds(180); @@ -1326,9 +1458,17 @@ private async Task PushBlobToNugetFeed(TargetFeedConfig feedConfig, string local feedConfig.Token)))); try { - await PushNugetPackageAsync(feedConfig, httpClient, localBlobPath, id, - version, feedAccount, feedVisibility, feedName); - } + await PushNugetPackageAsync( + feedConfig, + httpClient, + localBlobPath, + id, + version, + feedAccount, + feedVisibility, + feedName); + } + catch (Exception e) { Log.LogErrorFromException(e); @@ -1365,7 +1505,11 @@ private async Task PublishPackagesToAzureStorageNugetByDownloadingEntireFolderAs packagesToPublish .ToList() - .ForEach(package => TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + .ForEach(package => TryAddAssetLocation( + package.Id, + package.Version, + buildAssets, + feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed)); await blobFeedAction.PushToFeedAsync(packages, pushOptions); @@ -1376,7 +1520,12 @@ private async Task PublishPackagesToAzureStorageUsingStreamingAsync( Dictionary> buildAssets, TargetFeedConfig feedConfig) { - string containerId = await GetContainerIdAsync(); + string containerId = await GetContainerIdAsync(ArtifactName.PackageArtifacts); + + if (Log.HasLoggedErrors) + { + return; + } var blobFeedAction = CreateBlobFeedAction(feedConfig); var pushOptions = new PushOptions @@ -1385,6 +1534,7 @@ private async Task PublishPackagesToAzureStorageUsingStreamingAsync( PassIfExistingItemIdentical = true }; string localPackagePath = ""; + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true); foreach (var package in packagesToPublish) @@ -1392,8 +1542,14 @@ private async Task PublishPackagesToAzureStorageUsingStreamingAsync( string temporaryPackageDirectory = CreateTemporaryDirectory(); var packageFilename = $"{package.Id}.{package.Version}.nupkg"; localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - await DownloadFileAsync(client, PackageArtifacts, containerId, packageFilename, + + await DownloadFileAsync( + client, + ArtifactName.PackageArtifacts, + containerId, + packageFilename, localPackagePath); + if (!File.Exists(localPackagePath)) { Log.LogError( @@ -1401,15 +1557,22 @@ await DownloadFileAsync(client, PackageArtifacts, containerId, packageFilename, return; } - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, + TryAddAssetLocation( + package.Id, + package.Version, + buildAssets, + feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); + IEnumerable items = new List(); items.ToList().Add(localPackagePath); Log.LogMessage(MessageImportance.Low, $"START pushing {packageFilename} to feed"); + try { await blobFeedAction.PushAsync(items, pushOptions); } + catch (Exception e) { Log.LogErrorFromException(e); @@ -1425,22 +1588,21 @@ private async Task PublishPackagesToAzureStorageNugetFeedAsync( Dictionary> buildAssets, TargetFeedConfig feedConfig) { - if (!UseStreamingPublishing) + if (UseStreamingPublishing) { - await PublishPackagesToAzureStorageNugetByDownloadingEntireFolderAsync(packagesToPublish, buildAssets, - feedConfig); + await PublishPackagesToAzureStorageUsingStreamingAsync(packagesToPublish, buildAssets, feedConfig); } + else { - await PublishPackagesToAzureStorageUsingStreamingAsync(packagesToPublish, buildAssets, - feedConfig); + await PublishPackagesToAzureStorageNugetByDownloadingEntireFolderAsync(packagesToPublish, buildAssets, feedConfig); } } /// /// Creates a temporary directory /// - /// + /// Path to the directory created public string CreateTemporaryDirectory() { string temporaryDirectory = @@ -1454,20 +1616,26 @@ private async Task PublishBlobsToAzureStorageNugetFeedAsync( Dictionary> buildAssets, TargetFeedConfig feedConfig) { - if (!UseStreamingPublishing) + if (UseStreamingPublishing) { - await PublishBlobsToAzureStorageNugetAsync(blobsToPublish, buildAssets, feedConfig); + await PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync(blobsToPublish, buildAssets, feedConfig); } + else { - await PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync(blobsToPublish, buildAssets, - feedConfig); + await PublishBlobsToAzureStorageNugetAsync(blobsToPublish, buildAssets, feedConfig); } if (LinkManager == null) { - LinkManager = new LatestLinksManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, AkaMSGroupOwner, - AkaMSCreatedBy, AkaMsOwners, Log); + LinkManager = new LatestLinksManager( + AkaMSClientId, + AkaMSClientSecret, + AkaMSTenant, + AkaMSGroupOwner, + AkaMSCreatedBy, + AkaMsOwners, + Log); } // The latest links should be updated only after the publishing is complete, to avoid @@ -1481,7 +1649,12 @@ private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync( Dictionary> buildAssets, TargetFeedConfig feedConfig) { - string containerId = await GetContainerIdAsync(); + string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts); + + if (Log.HasLoggedErrors) + { + return; + } var blobFeedAction = CreateBlobFeedAction(feedConfig); var pushOptions = new PushOptions { @@ -1495,8 +1668,14 @@ private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync( string temporaryBlobDirectory = CreateTemporaryDirectory(); var fileName = Path.GetFileName(blob.Id); var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); - await DownloadFileAsync(client, "BlobArtifacts", containerId, fileName, + + await DownloadFileAsync( + client, + ArtifactName.BlobArtifacts, + containerId, + fileName, localBlobPath); + if (!File.Exists(localBlobPath)) { Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); @@ -1507,8 +1686,14 @@ await DownloadFileAsync(client, "BlobArtifacts", containerId, fileName, { {"RelativeBlobPath", blob.Id} }); - TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + + TryAddAssetLocation( + blob.Id, + assetVersion: null, + buildAssets, + feedConfig, AddAssetLocationToAssetAssetLocationType.Container); + await blobFeedAction.UploadAssetAsync(item, pushOptions, null); DeleteTemporaryFile(localBlobPath); DeleteTemporaryDirectory(temporaryBlobDirectory); @@ -1596,10 +1781,15 @@ private BlobFeedAction CreateBlobFeedAction(TargetFeedConfig feedConfig) } } - private async Task PushPackageToNugetFeed(TargetFeedConfig feedConfig, string localPackagePath, string id, + private async Task PushPackageToNugetFeed( + HttpClient httpClient, + TargetFeedConfig feedConfig, + string localPackagePath, + string id, string version) { var parsedUri = Regex.Match(feedConfig.TargetURL, PublishingConstants.AzDoNuGetFeedPattern); + if (!parsedUri.Success) { Log.LogError( @@ -1611,17 +1801,15 @@ private async Task PushPackageToNugetFeed(TargetFeedConfig feedConfig, string lo string feedVisibility = parsedUri.Groups["visibility"].Value; string feedName = parsedUri.Groups["feed"].Value; - using HttpClient httpClient = new HttpClient(new HttpClientHandler - {CheckCertificateRevocationList = true}); - - httpClient.Timeout = TimeSpan.FromSeconds(180); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( - "Basic", - Convert.ToBase64String( - Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); - await PushNugetPackageAsync(feedConfig, httpClient, localPackagePath, id, + await PushNugetPackageAsync( + feedConfig, + httpClient, + localPackagePath, + id, version, - feedAccount, feedVisibility, feedName); + feedAccount, + feedVisibility, + feedName); } /// From 90bafded2bce7f41fbb93ff21195814ab77d3a9a Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 1 Apr 2021 01:15:45 -0700 Subject: [PATCH 24/32] minor changes --- eng/publishing/v3/publish-assets.yml | 19 ------------------- .../src/PublishArtifactsInManifestBase.cs | 7 +++---- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml index 0a08d5f2463..af61f8bcf80 100644 --- a/eng/publishing/v3/publish-assets.yml +++ b/eng/publishing/v3/publish-assets.yml @@ -27,25 +27,6 @@ jobs: pool: vmImage: 'windows-2019' steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Assets - continueOnError: true - enabled: false - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: 'specific' - itemPattern: | - PackageArtifacts/** - BlobArtifacts/** - AssetManifests/** - PdbArtifacts/** - downloadPath: '$(Build.ArtifactStagingDirectory)' - checkDownloadedFiles: true - - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index d4c81b95724..f105a0d9555 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -394,7 +394,6 @@ public async Task PublishSymbolsUsingStreamingAsync( Dictionary serversToPublish = GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); - IEnumerable filesToSymbolServer = null; using (HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true)) { string localSymbolPath = ""; @@ -422,7 +421,7 @@ await PublishSymbolsHelper.PublishAsync( serverPath, token, symbolFiles, - filesToSymbolServer, + null, null, ExpirationInDays, false, @@ -452,6 +451,7 @@ await PublishSymbolsHelper.PublishAsync( } // publishing pdb artifacts + IEnumerable filesToSymbolServer = null; if (Directory.Exists(pdbArtifactsBasePath)) { var pdbEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.pdb", @@ -468,6 +468,7 @@ await PublishSymbolsHelper.PublishAsync( var serverPath = server.Key; var token = server.Value; symbolLog.AppendLine($"Publishing pdbFiles to {serverPath}:"); + try { await PublishSymbolsHelper.PublishAsync( @@ -485,7 +486,6 @@ await PublishSymbolsHelper.PublishAsync( false, true); } - catch (Exception ex) { Log.LogError(ex.Message); @@ -609,7 +609,6 @@ await PublishSymbolsHelper.PublishAsync( false, true); } - catch (Exception ex) { Log.LogError(ex.Message); From 41244b97717cf28e20ac65324610014f839dcd7d Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 1 Apr 2021 01:39:29 -0700 Subject: [PATCH 25/32] fix: --- .../src/PublishArtifactsInManifestBase.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index f105a0d9555..cf6b64ad5f4 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -5,6 +5,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; +using System.ComponentModel; using System.IO; using System.Linq; using System.Net; @@ -170,9 +171,13 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities private enum ArtifactName { + [Description("PackageArtifacts")] PackageArtifacts, + + [Description("BlobArtifacts")] BlobArtifacts } + public override bool Execute() { return ExecuteAsync().GetAwaiter().GetResult(); @@ -575,6 +580,7 @@ public async Task PublishSymbolsfromBlobArtifactsAsync( GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); IEnumerable filesToSymbolServer = null; + if (Directory.Exists(pdbArtifactsBasePath)) { var pdbEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.pdb", @@ -747,6 +753,7 @@ public HttpClient CreateAzdoClient( } string address = $"https://dev.azure.com/{accountName}/"; + if (!string.IsNullOrEmpty(projectName)) { address += $"{projectName}/"; @@ -787,7 +794,8 @@ private async Task GetContainerIdAsync(ArtifactName artifactName) foreach (var artifact in buildArtifacts.value) { - if (string.Equals(artifact.name, artifactName)) + ArtifactName name; + if (Enum.TryParse(artifact.name, out name) && name == artifactName) { string[] segment = artifact.resource.data.Split('/'); containerId = segment[1]; From d9f0469dcda564702969bc120af858ce7fc9304f Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 1 Apr 2021 11:42:54 -0700 Subject: [PATCH 26/32] remove error condition --- .../tools/SdkTasks/PublishArtifactsInManifest.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index 746bc65f582..28d23ef95c2 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -88,7 +88,6 @@ - @@ -109,6 +108,7 @@ $(InternalChecksumsAzureAccountKey) false false + $(BlobBasePath) Date: Thu, 1 Apr 2021 17:22:33 -0700 Subject: [PATCH 27/32] last few comments --- .../src/PublishArtifactsInManifestBase.cs | 249 +++++------------- .../src/PublishArtifactsInManifestV3.cs | 1 - 2 files changed, 64 insertions(+), 186 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index cf6b64ad5f4..5ca7e0c5d62 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -286,7 +286,6 @@ protected Task PersistPendingAssetLocationAsync(IMaestroApi client) /// /// Protect against accidental publishing of internal assets to non-internal feeds. /// - /// protected void CheckForInternalBuildsOnPublicFeeds(TargetFeedConfig feedConfig) { // If separated out for clarity. @@ -359,12 +358,12 @@ public void CheckForStableAssetsInNonIsolatedFeeds() /// /// Publishes files to symbol server(s) one by one using Azure api to download files /// - /// - /// - /// - /// - /// - /// + /// Path to dll and pdb files + /// Token to authenticate msdl + /// Token to authenticate symweb + /// Right now we do not add any files to this, so this is going to be null + /// Path to Symbol.nupkgs + /// If true, the special coreclr module indexed files like DBI, DAC and SOS are published /// Task public async Task PublishSymbolsUsingStreamingAsync( string pdbArtifactsBasePath, @@ -401,12 +400,10 @@ public async Task PublishSymbolsUsingStreamingAsync( using (HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true)) { - string localSymbolPath = ""; - foreach (var symbol in symbolsToPublish) { string temporarySymbolsDirectory = CreateTemporaryDirectory(); - localSymbolPath = Path.Combine(temporarySymbolsDirectory, symbol); + string localSymbolPath = Path.Combine(temporarySymbolsDirectory, symbol); await DownloadFileAsync(client, ArtifactName.BlobArtifacts, containerId, symbol, localSymbolPath); symbolLog.AppendLine($"Local Symbol path to downloaded file {localSymbolPath}"); @@ -445,7 +442,6 @@ await PublishSymbolsHelper.PublishAsync( DeleteTemporaryFile(localSymbolPath); DeleteTemporaryDirectory(temporarySymbolsDirectory); } - symbolLog.AppendLine( $"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false "); symbolLog.AppendLine($"Total number of symbol files : {symbolsToPublish.Count}"); @@ -515,7 +511,6 @@ await PublishSymbolsHelper.PublishAsync( /// Right now we do not add any files to this, so this is going to be null /// Path to Symbol.nupkgs /// If true, the special coreclr module indexed files like DBI, DAC and SOS are published - /// Task public async Task HandleSymbolPublishingAsync ( string pdbArtifactsBasePath, string msdlToken, @@ -535,7 +530,6 @@ await PublishSymbolsUsingStreamingAsync( publishSpecialClrFiles, buildAssets); } - else { await PublishSymbolsfromBlobArtifactsAsync( @@ -557,7 +551,6 @@ await PublishSymbolsfromBlobArtifactsAsync( /// Right now we do not add any files to this, so this is going to be null /// Path to Symbol.nupkgs /// If true, the special coreclr module indexed files like DBI, DAC and SOS are published - /// Task public async Task PublishSymbolsfromBlobArtifactsAsync( string pdbArtifactsBasePath, string msdlToken, @@ -626,7 +619,6 @@ await PublishSymbolsHelper.PublishAsync( symbolLog.Clear(); } } - else { Log.LogError("Temporary symbols directory does not exists."); @@ -691,15 +683,13 @@ protected async Task HandlePackagePublishingAsync(Dictionary FilterPackages(HashSet /// Name of the org /// If client is used for downloading the artifact then AutomaticDecompression has to be set, else it is not required - /// Azure project name + /// Azure devOps project name /// Default version is 6.0 /// HttpClient public HttpClient CreateAzdoClient( - string accountName, - bool setAutomaticDecompression, + string accountName, + bool setAutomaticDecompression, string projectName = null, string versionOverride = null) { @@ -753,7 +742,7 @@ public HttpClient CreateAzdoClient( } string address = $"https://dev.azure.com/{accountName}/"; - + if (!string.IsNullOrEmpty(projectName)) { address += $"{projectName}/"; @@ -813,6 +802,7 @@ private async Task GetContainerIdAsync(ArtifactName artifactName) Log.LogError("Container Id does not exists"); } + return containerId; } } @@ -829,12 +819,12 @@ private async Task DownloadFileAsync( HttpClient client, ArtifactName artifactName, string containerId, - string fileName, + string fileName, string path) { string uri = $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/_apis/resources/Containers/{containerId}?itemPath=/{artifactName}/{fileName}&isShallow=true&api-version={AzureApiVersionForFileDownload}"; - Log.LogMessage(MessageImportance.Low, $"download file uri = {uri}"); + Log.LogMessage(MessageImportance.Low, $"Download file uri = {uri}"); Exception mostRecentlyCaughtException = null; bool success = await RetryHandler.RunAsync(async attempt => { @@ -842,14 +832,14 @@ private async Task DownloadFileAsync( { CancellationTokenSource timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes)); - + using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); using HttpResponseMessage response = await client.GetAsync(uri, timeoutTokenSource.Token); response.EnsureSuccessStatusCode(); using var fs = new FileStream( - path, + path, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite); @@ -873,7 +863,7 @@ private async Task DownloadFileAsync( return false; } }).ConfigureAwait(false); - + if (!success) { throw new Exception( @@ -913,9 +903,9 @@ protected async Task HandleBlobPublishingAsync(Dictionary case FeedType.AzDoNugetFeed: publishTasks.Add( PublishBlobsToAzDoNugetFeedAsync( - filteredBlobs, - buildAssets, - feedConfig)); + filteredBlobs, + buildAssets, + feedConfig)); break; case FeedType.AzureStorageFeed: publishTasks.Add( @@ -931,7 +921,6 @@ protected async Task HandleBlobPublishingAsync(Dictionary } } } - else { Log.LogError($"No target feed configuration found for artifact category: '{category}'."); @@ -1044,19 +1033,20 @@ await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClien } TryAddAssetLocation( - package.Id, - package.Version, + package.Id, + package.Version, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); - await PushNugetPackageAsync(feed, - httpClient, - localPackagePath, - package.Id, + await PushNugetPackageAsync( + feed, + httpClient, + localPackagePath, + package.Id, package.Version, - feedAccount, - feedVisibility, + feedAccount, + feedVisibility, feedName); }); } @@ -1072,25 +1062,24 @@ private async Task PublishPackagesUsingStreamingToAzdoNugetAsync( { return; } - string localPackagePath = ""; - + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true); - + foreach (var package in packagesToPublish) { var packageFilename = $"{package.Id}.{package.Version}.nupkg"; string temporaryPackageDirectory = Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); EnsureTemporaryDirectoryExists(temporaryPackageDirectory); - localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - + string localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); + await DownloadFileAsync( client, ArtifactName.PackageArtifacts, containerId, packageFilename, localPackagePath); - + if (!File.Exists(localPackagePath)) { Log.LogError( @@ -1117,7 +1106,7 @@ await DownloadFileAsync( Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); await PushPackageToNugetFeed(httpClient, feedConfig, localPackagePath, package.Id, package.Version); - + File.Delete(localPackagePath); Directory.Delete(temporaryPackageDirectory); } @@ -1157,9 +1146,11 @@ public async Task PushNugetPackagesAsync( var parsedUri = Regex.Match(feedConfig.TargetURL, PublishingConstants.AzDoNuGetFeedPattern); if (!parsedUri.Success) { - Log.LogError($"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); + Log.LogError( + $"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); return; } + string feedAccount = parsedUri.Groups["account"].Value; string feedVisibility = parsedUri.Groups["visibility"].Value; string feedName = parsedUri.Groups["feed"].Value; @@ -1338,7 +1329,6 @@ private async Task PublishBlobsToAzDoNugetFeedAsync( { await PublishBlobsUsingStreamingToAzDoNugetAsync(packagesToPublish, buildAssets, feedConfig); } - else { await PublishBlobsFromBlobArtifactsToAzDoNugetAsync(packagesToPublish, buildAssets, feedConfig); @@ -1350,7 +1340,10 @@ private async Task PublishBlobsFromBlobArtifactsToAzDoNugetAsync( Dictionary> buildAssets, TargetFeedConfig feedConfig) { - await PushNugetPackagesAsync(blobsToPublish, feedConfig, maxClients: MaxClients, + await PushNugetPackagesAsync( + blobsToPublish, + feedConfig, + maxClients: MaxClients, async (feed, httpClient, blob, feedAccount, feedVisibility, feedName) => { if (TryAddAssetLocation( @@ -1406,14 +1399,21 @@ private async Task PublishBlobsUsingStreamingToAzDoNugetAsync( foreach (var blob in blobsToPublish) { - if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + if (TryAddAssetLocation( + blob.Id, + assetVersion: null, + buildAssets, + feedConfig, AddAssetLocationToAssetAssetLocationType.Container)) { string temporaryBlobDirectory = CreateTemporaryDirectory(); string fileName = Path.GetFileName(blob.Id); string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); - await DownloadFileAsync(client, ArtifactName.BlobArtifacts, containerId, + await DownloadFileAsync( + client, + ArtifactName.BlobArtifacts, + containerId, fileName, localBlobPath); @@ -1475,135 +1475,10 @@ await PushNugetPackageAsync( feedVisibility, feedName); } - catch (Exception e) { Log.LogErrorFromException(e); } - - } - - private async Task PublishPackagesToAzureStorageNugetByDownloadingEntireFolderAsync(HashSet packagesToPublish, - Dictionary> buildAssets, - TargetFeedConfig feedConfig) - { - var packages = packagesToPublish.Select(p => - { - var localPackagePath = Path.Combine(PackageAssetsBasePath, $"{p.Id}.{p.Version}.nupkg"); - if (!File.Exists(localPackagePath)) - { - Log.LogError($"Could not locate '{p.Id}.{p.Version}' at '{localPackagePath}'"); - } - return localPackagePath; - }); - - if (Log.HasLoggedErrors) - { - return; - } - - var blobFeedAction = CreateBlobFeedAction(feedConfig); - - var pushOptions = new PushOptions - { - AllowOverwrite = feedConfig.AllowOverwrite, - PassIfExistingItemIdentical = true - }; - - packagesToPublish - .ToList() - .ForEach(package => TryAddAssetLocation( - package.Id, - package.Version, - buildAssets, - feedConfig, - AddAssetLocationToAssetAssetLocationType.NugetFeed)); - - await blobFeedAction.PushToFeedAsync(packages, pushOptions); - } - - private async Task PublishPackagesToAzureStorageUsingStreamingAsync( - HashSet packagesToPublish, - Dictionary> buildAssets, - TargetFeedConfig feedConfig) - { - string containerId = await GetContainerIdAsync(ArtifactName.PackageArtifacts); - - if (Log.HasLoggedErrors) - { - return; - } - var blobFeedAction = CreateBlobFeedAction(feedConfig); - - var pushOptions = new PushOptions - { - AllowOverwrite = feedConfig.AllowOverwrite, - PassIfExistingItemIdentical = true - }; - string localPackagePath = ""; - - using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true); - - foreach (var package in packagesToPublish) - { - string temporaryPackageDirectory = CreateTemporaryDirectory(); - var packageFilename = $"{package.Id}.{package.Version}.nupkg"; - localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); - - await DownloadFileAsync( - client, - ArtifactName.PackageArtifacts, - containerId, - packageFilename, - localPackagePath); - - if (!File.Exists(localPackagePath)) - { - Log.LogError( - $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); - return; - } - - TryAddAssetLocation( - package.Id, - package.Version, - buildAssets, - feedConfig, - AddAssetLocationToAssetAssetLocationType.NugetFeed); - - IEnumerable items = new List(); - items.ToList().Add(localPackagePath); - Log.LogMessage(MessageImportance.Low, $"START pushing {packageFilename} to feed"); - - try - { - await blobFeedAction.PushAsync(items, pushOptions); - } - - catch (Exception e) - { - Log.LogErrorFromException(e); - } - - DeleteTemporaryFile(localPackagePath); - DeleteTemporaryDirectory(temporaryPackageDirectory); - } - } - - private async Task PublishPackagesToAzureStorageNugetFeedAsync( - HashSet packagesToPublish, - Dictionary> buildAssets, - TargetFeedConfig feedConfig) - { - if (UseStreamingPublishing) - { - await PublishPackagesToAzureStorageUsingStreamingAsync(packagesToPublish, buildAssets, feedConfig); - } - - else - { - await PublishPackagesToAzureStorageNugetByDownloadingEntireFolderAsync(packagesToPublish, buildAssets, feedConfig); - } } /// @@ -1627,7 +1502,6 @@ private async Task PublishBlobsToAzureStorageNugetFeedAsync( { await PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync(blobsToPublish, buildAssets, feedConfig); } - else { await PublishBlobsToAzureStorageNugetAsync(blobsToPublish, buildAssets, feedConfig); @@ -1644,10 +1518,11 @@ private async Task PublishBlobsToAzureStorageNugetFeedAsync( AkaMsOwners, Log); } - // The latest links should be updated only after the publishing is complete, to avoid // dead links in the interim. - await LinkManager.CreateOrUpdateLatestLinksAsync(blobsToPublish, feedConfig, + await LinkManager.CreateOrUpdateLatestLinksAsync( + blobsToPublish, + feedConfig, PublishingConstants.ExpectedFeedUrlSuffix.Length); } @@ -1743,7 +1618,11 @@ private async Task PublishBlobsToAzureStorageNugetAsync( blobsToPublish .ToList() - .ForEach(blob => TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, + .ForEach(blob => TryAddAssetLocation( + blob.Id, + assetVersion: null, + buildAssets, + feedConfig, AddAssetLocationToAssetAssetLocationType.Container)); await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients: MaxClients, pushOptions); diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index cb18fc0cd1e..8211b18df9a 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -192,7 +192,6 @@ public override async Task ExecuteAsync() string temporarySymbolsLocation = ""; if (!UseStreamingPublishing) { - temporarySymbolsLocation = Path.GetFullPath(Path.Combine(BlobAssetsBasePath, @"..\", "tempSymbols")); From 21fc09fe1b618ef69c7ee156d8282365aad4e8cd Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 1 Apr 2021 17:30:14 -0700 Subject: [PATCH 28/32] Removed unwanted comment --- .../src/PublishArtifactsInManifestBase.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 5ca7e0c5d62..e90d6c4e783 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -362,7 +362,6 @@ public void CheckForStableAssetsInNonIsolatedFeeds() /// Token to authenticate msdl /// Token to authenticate symweb /// Right now we do not add any files to this, so this is going to be null - /// Path to Symbol.nupkgs /// If true, the special coreclr module indexed files like DBI, DAC and SOS are published /// Task public async Task PublishSymbolsUsingStreamingAsync( From 86215a969976f586e08f3bd62ecc45398e7a969a Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Mon, 5 Apr 2021 15:59:55 -0700 Subject: [PATCH 29/32] review comment --- .../src/PublishArtifactsInManifestBase.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index e90d6c4e783..cfb8d8520ae 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -454,9 +454,13 @@ await PublishSymbolsHelper.PublishAsync( IEnumerable filesToSymbolServer = null; if (Directory.Exists(pdbArtifactsBasePath)) { - var pdbEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.pdb", + var pdbEntries = System.IO.Directory.EnumerateFiles( + pdbArtifactsBasePath, + "*.pdb", System.IO.SearchOption.AllDirectories); - var dllEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.dll", + var dllEntries = System.IO.Directory.EnumerateFiles( + pdbArtifactsBasePath, + "*.dll", System.IO.SearchOption.AllDirectories); filesToSymbolServer = pdbEntries.Concat(dllEntries); } @@ -575,9 +579,13 @@ public async Task PublishSymbolsfromBlobArtifactsAsync( if (Directory.Exists(pdbArtifactsBasePath)) { - var pdbEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.pdb", + var pdbEntries = System.IO.Directory.EnumerateFiles( + pdbArtifactsBasePath, + "*.pdb", System.IO.SearchOption.AllDirectories); - var dllEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.dll", + var dllEntries = System.IO.Directory.EnumerateFiles( + pdbArtifactsBasePath, + "*.dll", System.IO.SearchOption.AllDirectories); filesToSymbolServer = pdbEntries.Concat(dllEntries); } From 8de357ded1df662a6ecd2611a11bb505320f028d Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Tue, 6 Apr 2021 11:29:39 -0700 Subject: [PATCH 30/32] last change --- .../src/PublishArtifactsInManifestBase.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index cfb8d8520ae..e0bbbe0974f 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -773,6 +773,7 @@ public HttpClient CreateAzdoClient( /// Gets the container Id, that is going to be used in another API call to download the assets /// ContainerId is the same for PackageArtifacts and BlobArtifacts /// + /// If it is PackageArtifacts or BlobArtifacts /// ContainerId private async Task GetContainerIdAsync(ArtifactName artifactName) { @@ -818,7 +819,7 @@ private async Task GetContainerIdAsync(ArtifactName artifactName) /// Download artifact file using Azure API /// /// Azdo client - /// If it is PackageArtifacts or BlobArtifacts + /// If it is PackageArtifacts or BlobArtifacts /// ContainerId where the packageArtifact and BlobArtifacts are stored /// Name the file we are trying to download /// Path where the file is being downloaded From 8a5d431c8eb72191c158b02f5fd0bd0fe0b541d3 Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 8 Apr 2021 14:13:27 -0700 Subject: [PATCH 31/32] IO exception fix --- .../src/PublishArtifactsInManifestBase.cs | 132 ++++++++++-------- .../src/PublishArtifactsInManifestV3.cs | 10 +- .../src/common/GeneralUtils.cs | 4 +- 3 files changed, 83 insertions(+), 63 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index e0bbbe0974f..2655601a893 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -178,6 +178,8 @@ private enum ArtifactName BlobArtifacts } + private int TimeoutInSeconds = 180; + public override bool Execute() { return ExecuteAsync().GetAwaiter().GetResult(); @@ -403,9 +405,10 @@ public async Task PublishSymbolsUsingStreamingAsync( { string temporarySymbolsDirectory = CreateTemporaryDirectory(); string localSymbolPath = Path.Combine(temporarySymbolsDirectory, symbol); - + symbolLog.AppendLine($"Downloading symbol : {symbol} to {localSymbolPath}"); + await DownloadFileAsync(client, ArtifactName.BlobArtifacts, containerId, symbol, localSymbolPath); - symbolLog.AppendLine($"Local Symbol path to downloaded file {localSymbolPath}"); + symbolLog.AppendLine($"Successfully downloaded symbol : {symbol} to {localSymbolPath}"); List symbolFiles = new List(); symbolFiles.ToList().Add(localSymbolPath); @@ -438,7 +441,6 @@ await PublishSymbolsHelper.PublishAsync( } } - DeleteTemporaryFile(localSymbolPath); DeleteTemporaryDirectory(temporarySymbolsDirectory); } symbolLog.AppendLine( @@ -759,6 +761,7 @@ public HttpClient CreateAzdoClient( { BaseAddress = new Uri(address) }; + client.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( "Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", @@ -777,14 +780,21 @@ public HttpClient CreateAzdoClient( /// ContainerId private async Task GetContainerIdAsync(ArtifactName artifactName) { + string uri = + $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/{AzureProject}/_apis/build/builds/{BuildId}/artifacts?api-version={AzureDevOpsFeedsApiVersion}"; + Exception mostRecentlyCaughtException = null; string containerId = ""; - using (HttpClient client = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject)) + bool success = await RetryHandler.RunAsync(async attempt => { - string uri = - $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/{AzureProject}/_apis/build/builds/{BuildId}/artifacts?api-version={AzureDevOpsFeedsApiVersion}"; try { - using HttpResponseMessage response = await client.GetAsync(uri); + CancellationTokenSource timeoutTokenSource = + new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes)); + + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject); + using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); + using HttpResponseMessage response = await client.GetAsync(uri, timeoutTokenSource.Token); + response.EnsureSuccessStatusCode(); string responseBody = await response.Content.ReadAsStringAsync(); BuildArtifacts buildArtifacts = JsonConvert.DeserializeObject(responseBody); @@ -796,23 +806,30 @@ private async Task GetContainerIdAsync(ArtifactName artifactName) { string[] segment = artifact.resource.data.Split('/'); containerId = segment[1]; - break; + return true; } } + return false; } - catch (Exception e) + catch (HttpRequestException toStore) { - Log.LogErrorFromException(e); + mostRecentlyCaughtException = toStore; + return false; } + }).ConfigureAwait(false); - if (string.IsNullOrEmpty(containerId)) - { - Log.LogError("Container Id does not exists"); - - } + if (string.IsNullOrEmpty(containerId)) + { + Log.LogError("Container Id does not exists"); + } - return containerId; + if (!success) + { + throw new Exception( + $"Failed to get container id after {RetryHandler.MaxAttempts} attempts. See inner exception for details, {mostRecentlyCaughtException}"); } + + return containerId; } /// @@ -1080,6 +1097,7 @@ private async Task PublishPackagesUsingStreamingToAzdoNugetAsync( Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); EnsureTemporaryDirectoryExists(temporaryPackageDirectory); string localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); + Log.LogMessage(MessageImportance.Low, $"Downloading package : {packageFilename} to {localPackagePath}"); await DownloadFileAsync( client, @@ -1094,6 +1112,7 @@ await DownloadFileAsync( $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); return; } + Log.LogMessage(MessageImportance.Low, $"Successfully downloaded package : {packageFilename} to {localPackagePath}"); TryAddAssetLocation( package.Id, @@ -1107,7 +1126,7 @@ await DownloadFileAsync( CheckCertificateRevocationList = true }); - httpClient.Timeout = TimeSpan.FromSeconds(180); + httpClient.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( "Basic", Convert.ToBase64String( @@ -1115,8 +1134,7 @@ await DownloadFileAsync( await PushPackageToNugetFeed(httpClient, feedConfig, localPackagePath, package.Id, package.Version); - File.Delete(localPackagePath); - Directory.Delete(temporaryPackageDirectory); + DeleteTemporaryDirectory(localPackagePath); } } @@ -1373,12 +1391,13 @@ await PushNugetPackagesAsync( string id; string version; // Determine package ID and version by asking the nuget libraries - using var packageReader = new PackageArchiveReader(localBlobPath); + using (var packageReader = new PackageArchiveReader(localBlobPath)) + { + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); + } - PackageIdentity packageIdentity = packageReader.GetIdentity(); - id = packageIdentity.Id; - version = packageIdentity.Version.ToString(); - await PushNugetPackageAsync( feed, httpClient, @@ -1417,7 +1436,8 @@ private async Task PublishBlobsUsingStreamingToAzDoNugetAsync( string temporaryBlobDirectory = CreateTemporaryDirectory(); string fileName = Path.GetFileName(blob.Id); string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); - + Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}"); + await DownloadFileAsync( client, ArtifactName.BlobArtifacts, @@ -1429,21 +1449,23 @@ await DownloadFileAsync( { Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); } + Log.LogMessage(MessageImportance.Low, $"Successfully downloaded blob : {fileName} to {localBlobPath}"); + string id; string version; - using var packageReader = new PackageArchiveReader(localBlobPath); + using (var packageReader = new PackageArchiveReader(localBlobPath)) + { + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); + } - PackageIdentity packageIdentity = packageReader.GetIdentity(); - id = packageIdentity.Id; - version = packageIdentity.Version.ToString(); - await PushBlobToNugetFeed( feedConfig, localBlobPath, id, version); - DeleteTemporaryFile(localBlobPath); DeleteTemporaryDirectory(temporaryBlobDirectory); } } @@ -1558,7 +1580,8 @@ private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync( string temporaryBlobDirectory = CreateTemporaryDirectory(); var fileName = Path.GetFileName(blob.Id); var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); - + Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}"); + await DownloadFileAsync( client, ArtifactName.BlobArtifacts, @@ -1570,6 +1593,7 @@ await DownloadFileAsync( { Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); } + Log.LogMessage(MessageImportance.Low, $"Successfully downloaded blob : {fileName} to {localBlobPath}"); var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, new Dictionary @@ -1585,7 +1609,6 @@ await DownloadFileAsync( AddAssetLocationToAssetAssetLocationType.Container); await blobFeedAction.UploadAssetAsync(item, pushOptions, null); - DeleteTemporaryFile(localBlobPath); DeleteTemporaryDirectory(temporaryBlobDirectory); } } @@ -1741,42 +1764,33 @@ public void DeleteTemporaryFiles(string temporaryLocation) } } - /// - /// Delete the files after publishing, this is part of cleanup - /// - /// - public void DeleteTemporaryFile(string filePath) - { - try - { - if (File.Exists(filePath)) - { - Log.LogMessage($"Going to delete the following file {filePath}"); - File.Delete(filePath); - } - } - catch (Exception ex) - { - Log.LogWarning(ex.Message); - } - } - /// /// Deletes the temporary folder, this is part of clean up /// /// public void DeleteTemporaryDirectory(string temporaryLocation) { + var attempts = 0; if (Directory.Exists(temporaryLocation)) { - try - { - Directory.Delete(temporaryLocation); - } - catch (Exception ex) + do { - Log.LogWarning(ex.Message); + try + { + attempts++; + Log.LogMessage(MessageImportance.Low, $"Deleting directory : {temporaryLocation}"); + Directory.Delete(temporaryLocation, true); + break; + } + catch (Exception ex) + { + if (attempts == MaxRetryCount) + Log.LogMessage(MessageImportance.Low, $"Unable to delete the directory because of {ex.Message} after {attempts} attempts."); + } + Log.LogMessage(MessageImportance.Low, $"Retrying to delete {temporaryLocation}, attempt number {attempts}"); + Task.Delay(RetryDelayMilliseconds).Wait(); } + while (true); } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index 8211b18df9a..4228c42d701 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -206,8 +206,14 @@ public override async Task ExecuteAsync() await Task.WhenAll(new Task[] { HandlePackagePublishingAsync(buildAssets), HandleBlobPublishingAsync(buildAssets), - HandleSymbolPublishingAsync(PdbArtifactsBasePath, MsdlToken, - SymWebToken, SymbolPublishingExclusionsFile, PublishSpecialClrFiles, buildAssets, temporarySymbolsLocation) + HandleSymbolPublishingAsync( + PdbArtifactsBasePath, + MsdlToken, + SymWebToken, + SymbolPublishingExclusionsFile, + PublishSpecialClrFiles, + buildAssets, + temporarySymbolsLocation) }); DeleteTemporaryFiles(temporarySymbolsLocation); diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/GeneralUtils.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/GeneralUtils.cs index 90b58e851a3..3e039ac74b8 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/GeneralUtils.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/GeneralUtils.cs @@ -134,13 +134,13 @@ public static async Task CompareStreamsAsync(Stream localFileStream, Strea /// the streams make no guarantee that they will return a full block each time when read operations are performed, so we /// must be sure to only compare the minimum number of bytes returned. /// - public static Task CompareLocalPackageToFeedPackage( + public static async Task CompareLocalPackageToFeedPackage( string localPackageFullPath, string packageContentUrl, HttpClient client, TaskLoggingHelper log) { - return CompareLocalPackageToFeedPackage( + return await CompareLocalPackageToFeedPackage( localPackageFullPath, packageContentUrl, client, From 07f63c4f85a223b5daba6a5ba4db98c6369164ba Mon Sep 17 00:00:00 2001 From: Epsitha Ananth Date: Thu, 8 Apr 2021 15:24:27 -0700 Subject: [PATCH 32/32] test fix --- .../PublishToSymbolServerTest.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs index 0133c110406..926fa188d10 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs @@ -93,7 +93,11 @@ public void TemporarySymbolDirectoryDoesNotExists() [Fact] public void TemporarySymbolsDirectoryTest() { - var publishTask = new PublishArtifactsInManifestV3(); + var buildEngine = new MockBuildEngine(); + var publishTask = new PublishArtifactsInManifestV3() + { + BuildEngine = buildEngine, + }; var path = TestInputs.GetFullPath("Test"); publishTask.EnsureTemporaryDirectoryExists(path); Assert.True(Directory.Exists(path));