Publish artifacts and symbols one file at a time (dotnet#7086)

* Improve publishing performance

eng/publishing/v3/publish-assets.yml

@@ -22,12 +22,15 @@ jobs:
       value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
     - name: AzDOBuildId
       value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+    - name: AzDOAccount
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildAccount'] ]
   pool:
     vmImage: 'windows-2019'
   steps:
     - task: DownloadBuildArtifacts@0
       displayName: Download Build Assets
       continueOnError: true
+      enabled: true
       inputs:
         buildType: specific
         buildVersionToDownload: specific
@@ -36,12 +39,9 @@ jobs:
         buildId: $(AzDOBuildId)
         downloadType: 'specific'
         itemPattern: |
-          PackageArtifacts/**
-          BlobArtifacts/**
           AssetManifests/**
           PdbArtifacts/**
         downloadPath: '$(Build.ArtifactStagingDirectory)'
-        checkDownloadedFiles: true
 
     - task: NuGetToolInstaller@1
       displayName: 'Install NuGet.exe'
@@ -87,7 +87,12 @@ jobs:
           /p:MsdlToken=$(microsoft-symbol-server-pat)
           /p:SymWebToken=$(symweb-symbol-server-pat)
           /p:BuildQuality='${{ parameters.buildQuality }}'
-
+          /p:AzdoApiToken='$(dn-bot-all-orgs-build-rw-code-rw)'
+          /p:ArtifactsBasePath='$(Build.ArtifactStagingDirectory)/'
+          /p:BuildId='$(AzDOBuildId)'
+          /p:AzureDevOpsOrg='$(AzDOAccount)'
+          /p:AzureProject='$(AzDOProjectName)'
+          /p:UseStreamingPublishing='true'
     - template: /eng/common/templates/steps/publish-logs.yml
       parameters:
         StageLabel: '${{ parameters.stageName }}'

eng/publishing/v3/setup-maestro-vars.yml

@@ -38,6 +38,7 @@ jobs:
             $AzureDevOpsProject = $buildInfo.azureDevOpsProject
             $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId
             $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId
+            $AzureDevOpsAccount = $buildInfo.azureDevOpsAccount
 
             Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId"
             Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels"
@@ -46,6 +47,7 @@ jobs:
             Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject"
             Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId"
             Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId"
+            Write-Host "##vso[task.setvariable variable=AzDOBuildAccount;isOutput=true]$AzureDevOpsAccount"
           }
           catch {
             Write-Host $_

src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj

@@ -23,6 +23,10 @@
       - SymWebToken                             : Token used to publish symbol blobs, dll and pdb to Symweb
       - SymbolPublishingExclusionsFile          : Files that have to be excluded from symbol publishing 
       - PdbArtifactsBasePath                    : Path to dlls and pdbs
+      - AzdoApiToken                            : Token used to call the Azure Api to download artifacts and symbols
+      - ArtifactsBasePath                       : Staging directory 
+      - UseStreamingPublishing                  : When set to true it will use Azure Api to download the artifacts and symbols using streaming,
+                                                  else all the artifacts and symbols are downloaded before publishing.
 
       Optional aka.ms link generation parameters:
         - AkaMSClientId                         : Client ID for the aka.ms AD application
@@ -103,6 +107,8 @@
       <ChecksumsFeedKey>$(ChecksumsAzureAccountKey)</ChecksumsFeedKey>
       <InternalChecksumsFeedKey>$(InternalChecksumsAzureAccountKey)</InternalChecksumsFeedKey>
       <AllowFeedOverrides Condition="'$(AllowFeedOverrides)' == ''">false</AllowFeedOverrides>
+      <UseStreamingPublishing Condition="'$(UseStreamingPublishing)' == ''">false</UseStreamingPublishing>
+      <ArtifactsBasePath Condition="'$(ArtifactsBasePath)' == ''">$(BlobBasePath)</ArtifactsBasePath>
     </PropertyGroup>
 
     <Error
@@ -147,7 +153,14 @@
       ChecksumsFeedOverride="$(ChecksumsFeedOverride)"
       TransportFeedOverride="$(TransportFeedOverride)"
       ShippingFeedOverride="$(ShippingFeedOverride)"
-      SymbolsFeedOverride="$(SymbolsFeedOverride)" />
+      SymbolsFeedOverride="$(SymbolsFeedOverride)" 
+      AzdoApiToken="$(AzdoApiToken)"
+      ArtifactsBasePath="$(ArtifactsBasePath)"
+      BuildId="$(BuildId)"
+      AzureDevOpsOrg="$(AzureDevOpsOrg)"
+      AzureProject="$(AzureProject)"
+      UseStreamingPublishing="$(UseStreamingPublishing)"/>
+
   </Target>
 
   <ItemGroup>

src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs

@@ -1,8 +1,8 @@
 using System.Collections.Generic;
 using System.IO;
-using Castle.DynamicProxy.Generators.Emitters.SimpleAST;
 using Microsoft.Arcade.Test.Common;
 using Microsoft.DotNet.Build.Tasks.Feed.Model;
+using Microsoft.DotNet.Maestro.Client.Models;
 using Xunit;
 
 namespace Microsoft.DotNet.Build.Tasks.Feed.Tests
@@ -85,18 +85,23 @@ public void TemporarySymbolDirectoryDoesNotExists()
                 BuildEngine = buildEngine,
             };
             var path = TestInputs.GetFullPath("Symbol");
-            var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", path, false);
+            var buildAsset = new Dictionary<string, HashSet<Asset>>();
+            var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, buildAsset, path);
             Assert.True(task.Log.HasLoggedErrors);
         }
 
         [Fact]
         public void TemporarySymbolsDirectoryTest()
         {
-            var publishTask = new PublishArtifactsInManifestV3();
+            var buildEngine = new MockBuildEngine();
+            var publishTask = new PublishArtifactsInManifestV3()
+            {
+                BuildEngine = buildEngine,
+            };
             var path = TestInputs.GetFullPath("Test");
-            publishTask.EnsureTemporarySymbolDirectoryExists(path);
+            publishTask.EnsureTemporaryDirectoryExists(path);
             Assert.True(Directory.Exists(path));
-            publishTask.DeleteSymbolTemporaryDirectory(path);
+            publishTask.DeleteTemporaryDirectory(path);
             Assert.False(Directory.Exists(path));
         }
 
@@ -134,6 +139,5 @@ public void PublishSymbolApiIsCalledTest()
                 false,
                 false).IsCompleted);
         }
-
     }
 }

src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs

@@ -1,23 +1,21 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
 using Microsoft.Build.Framework;
 using Microsoft.DotNet.Build.CloudTestTasks;
 using Microsoft.WindowsAzure.Storage;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Newtonsoft.Json.Serialization;
-using NuGet.Packaging;
 using NuGet.Packaging.Core;
 using Sleet;
-using System;
-using System.Collections.Generic;
-using System.ComponentModel;
-using System.IO;
-using System.Linq;
-using System.Text.RegularExpressions;
-using System.Threading;
-using System.Threading.Tasks;
 using MSBuild = Microsoft.Build.Utilities;
 
 namespace Microsoft.DotNet.Build.Tasks.Feed
@@ -124,26 +122,24 @@ public async Task<bool> PushItemsToFeedAsync(
             return !Log.HasLoggedErrors;
         }
 
-        public async Task PublishToFlatContainerAsync(IEnumerable<ITaskItem> taskItems, int maxClients, PushOptions pushOptions)
+        public async Task PublishToFlatContainerAsync(IEnumerable<ITaskItem> taskItems, int maxClients,
+            PushOptions pushOptions)
         {
             if (taskItems.Any())
             {
                 using (var clientThrottle = new SemaphoreSlim(maxClients, maxClients))
                 {
                     await System.Threading.Tasks.Task.WhenAll(taskItems.Select(
-                        item =>
-                        {
-                            return UploadAssetAsync(item, clientThrottle, pushOptions);
-                        }
+                        item => { return UploadAssetAsync(item, pushOptions, clientThrottle); }
                     ));
                 }
             }
         }
 
         public async Task UploadAssetAsync(
             ITaskItem item,
-            SemaphoreSlim clientThrottle,
-            PushOptions options)
+            PushOptions options,
+            SemaphoreSlim clientThrottle = null)
         {
             string relativeBlobPath = item.GetMetadata("RelativeBlobPath");
 
@@ -154,58 +150,72 @@ public async Task UploadAssetAsync(
                 relativeBlobPath = $"{recursiveDir}{fileName}";
             }
 
-            relativeBlobPath = $"{RelativePath}{relativeBlobPath}".Replace("\\", "/");
-
-            if (relativeBlobPath.Contains("//"))
+            if (!string.IsNullOrEmpty(relativeBlobPath))
             {
-                Log.LogError(
-                    $"Item '{item.ItemSpec}' RelativeBlobPath contains virtual directory " +
-                    $"without name (double forward slash): '{relativeBlobPath}'");
-                return;
-            }
+                relativeBlobPath = $"{RelativePath}{relativeBlobPath}".Replace("\\", "/");
 
-            Log.LogMessage($"Uploading {relativeBlobPath}");
+                if (relativeBlobPath.StartsWith("//"))
+                {
+                    Log.LogError(
+                        $"Item '{item.ItemSpec}' RelativeBlobPath contains virtual directory " +
+                        $"without name (double forward slash): '{relativeBlobPath}'");
+                    return;
+                }
 
-            await clientThrottle.WaitAsync();
+                Log.LogMessage($"Uploading {relativeBlobPath}");
 
-            try
-            {
-                AzureStorageUtils blobUtils = new AzureStorageUtils(AccountName, AccountKey, ContainerName);
+                if (clientThrottle != null)
+                {
+                    await clientThrottle.WaitAsync();
+                }
 
-                if (!options.AllowOverwrite && await blobUtils.CheckIfBlobExistsAsync(relativeBlobPath))
+                try
                 {
-                    if (options.PassIfExistingItemIdentical)
+                    AzureStorageUtils blobUtils = new AzureStorageUtils(AccountName, AccountKey, ContainerName);
+
+                    if (!options.AllowOverwrite && await blobUtils.CheckIfBlobExistsAsync(relativeBlobPath))
                     {
-                        if (!await blobUtils.IsFileIdenticalToBlobAsync(item.ItemSpec, relativeBlobPath))
+                        if (options.PassIfExistingItemIdentical)
                         {
-                            Log.LogError(
-                                $"Item '{item}' already exists with different contents " +
-                                $"at '{relativeBlobPath}'");
+                            if (!await blobUtils.IsFileIdenticalToBlobAsync(item.ItemSpec, relativeBlobPath))
+                            {
+                                Log.LogError(
+                                    $"Item '{item}' already exists with different contents " +
+                                    $"at '{relativeBlobPath}'");
+                            }
+                        }
+                        else
+                        {
+                            Log.LogError($"Item '{item}' already exists at '{relativeBlobPath}'");
                         }
                     }
                     else
                     {
-                        Log.LogError($"Item '{item}' already exists at '{relativeBlobPath}'");
+                        using (FileStream stream =
+                            new FileStream(item.ItemSpec, FileMode.Open, FileAccess.Read, FileShare.Read))
+                        {
+                            Log.LogMessage($"Uploading {item} to {relativeBlobPath}.");
+                            await blobUtils.UploadBlockBlobAsync(item.ItemSpec, relativeBlobPath, stream);
+                        }
                     }
                 }
-                else
+                catch (Exception exc)
                 {
-                    using (FileStream stream =
-                        new FileStream(item.ItemSpec, FileMode.Open, FileAccess.Read, FileShare.Read))
+                    Log.LogError(
+                        $"Unable to upload to {relativeBlobPath} in Azure Storage account {AccountName}/{ContainerName} due to {exc}.");
+                    throw;
+                }
+                finally
+                {
+                    if (clientThrottle != null)
                     {
-                        Log.LogMessage($"Uploading {item} to {relativeBlobPath}.");
-                        await blobUtils.UploadBlockBlobAsync(item.ItemSpec, relativeBlobPath, stream);
+                        clientThrottle.Release();
                     }
                 }
             }
-            catch (Exception exc)
-            {
-                Log.LogError($"Unable to upload to {relativeBlobPath} in Azure Storage account {AccountName}/{ContainerName} due to {exc}.");
-                throw;
-            }
-            finally
+            else
             {
-                clientThrottle.Release();
+                Log.LogError($"Relative blob path is empty.");
             }
         }
 
@@ -332,7 +342,7 @@ private ISleetFileSystem GetAzureFileSystem(LocalCache fileCache)
             }
         }
 
-        private async Task<bool> PushAsync(IEnumerable<string> items, PushOptions options)
+        public async Task<bool> PushAsync(IEnumerable<string> items, PushOptions options)
         {
             LocalSettings settings = GetSettings();
             SleetLogger log = new SleetLogger(Log, NuGet.Common.LogLevel.Verbose);

src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs

@@ -183,6 +183,21 @@ public string BuildQuality
             get { return _buildQuality.GetDescription(); }
             set { Enum.TryParse<PublishingConstants.BuildQuality>(value, true, out _buildQuality); }
         }
+        public string AzdoApiToken {get; set;}
+
+        public string ArtifactsBasePath { get; set;}
+
+        public string BuildId { get; set; }
+
+        public string AzureProject { get; set; }
+
+        public string AzureDevOpsOrg { get; set; }
+
+        /// <summary>
+        /// If true, uses Azdo Api to download artifacts and symbols files one file at a time during publishing process.
+        /// If it is set to false, then artifacts and symbols are downloaded in PackageArtifacts and BlobArtifacts directory before publishing. 
+        /// </summary>
+        public bool UseStreamingPublishing { get; set; } = false;
 
         /// <summary>
         /// Just an internal flag to keep track whether we published assets via a V3 manifest or not.
@@ -372,7 +387,13 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui
                 ChecksumsFeedOverride = this.ChecksumsFeedOverride,
                 ShippingFeedOverride = this.ShippingFeedOverride,
                 TransportFeedOverride = this.TransportFeedOverride,
-                SymbolsFeedOverride = this.SymbolsFeedOverride
+                SymbolsFeedOverride = this.SymbolsFeedOverride,
+                ArtifactsBasePath =  this.ArtifactsBasePath,
+                AzdoApiToken = this.AzdoApiToken,
+                BuildId = this.BuildId,
+                AzureProject = this.AzureProject,
+                AzureDevOpsOrg = this.AzureDevOpsOrg,
+                UseStreamingPublishing = this.UseStreamingPublishing
             };
         }
     }