Publish artifacts and symbols one file at a time

eng/publishing/v3/publish-assets.yml

@@ -22,12 +22,15 @@ jobs:
       value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
     - name: AzDOBuildId
       value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+    - name: AzDOAccount
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildAccount'] ]
   pool:
     vmImage: 'windows-2019'
   steps:
     - task: DownloadBuildArtifacts@0
       displayName: Download Build Assets
       continueOnError: true
+      enabled: false
       inputs:
         buildType: specific
         buildVersionToDownload: specific
@@ -43,6 +46,22 @@ jobs:
         downloadPath: '$(Build.ArtifactStagingDirectory)'
         checkDownloadedFiles: true
 
+    - task: DownloadBuildArtifacts@0
+      displayName: Download Build Assets
+      continueOnError: true
+      enabled: true
+      inputs:
+        buildType: specific
+        buildVersionToDownload: specific
+        project: $(AzDOProjectName)
+        pipeline: $(AzDOPipelineId)
+        buildId: $(AzDOBuildId)
+        downloadType: 'specific'
+        itemPattern: |
+          AssetManifests/**
+          PdbArtifacts/**
+        downloadPath: '$(Build.ArtifactStagingDirectory)'
+
     - task: NuGetToolInstaller@1
       displayName: 'Install NuGet.exe'
 
@@ -87,7 +106,12 @@ jobs:
           /p:MsdlToken=$(microsoft-symbol-server-pat)
           /p:SymWebToken=$(symweb-symbol-server-pat)
           /p:BuildQuality='${{ parameters.buildQuality }}'
-
+          /p:AzdoApiToken='$(dn-bot-dotnet-build-rw-code-rw)'
+          /p:StagingDir='$(Build.ArtifactStagingDirectory)/'
+          /p:BuildId='$(AzDOBuildId)'
+          /p:AzureDevOpsOrg='dnceng'
+          /p:AzureProject='$(AzDOProjectName)'
+          /p:UseApiOverride='true'
     - template: /eng/common/templates/steps/publish-logs.yml
       parameters:
         StageLabel: '${{ parameters.stageName }}'

eng/publishing/v3/setup-maestro-vars.yml

@@ -38,6 +38,7 @@ jobs:
             $AzureDevOpsProject = $buildInfo.azureDevOpsProject
             $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId
             $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId
+            $AzureDevOpsAccount = $buildInfo.azureDevOpsAccount
 
             Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId"
             Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels"
@@ -46,6 +47,7 @@ jobs:
             Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject"
             Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId"
             Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId"
+            Write-Host "##vso[task.setvariable variable=AzDOBuildAccount;isOutput=true]$AzureDevOpsAccount"
           }
           catch {
             Write-Host $_

src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj

@@ -138,7 +138,14 @@
       ChecksumsFeedOverride="$(ChecksumsFeedOverride)"
       TransportFeedOverride="$(TransportFeedOverride)"
       ShippingFeedOverride="$(ShippingFeedOverride)"
-      SymbolsFeedOverride="$(SymbolsFeedOverride)" />
+      SymbolsFeedOverride="$(SymbolsFeedOverride)" 
+      AzdoApiToken="$(AzdoApiToken)"
+      StagingDir="$(StagingDir)"
+      BuildId="$(BuildId)"
+      AzureDevOpsOrg="$(AzureDevOpsOrg)"
+      AzureProject="$(AzureProject)"
+      UseApiOverride="$(UseApiOverride)"/>
+
   </Target>
 
   <ItemGroup>

src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs

@@ -85,7 +85,7 @@ public void TemporarySymbolDirectoryDoesNotExists()
                 BuildEngine = buildEngine,
             };
             var path = TestInputs.GetFullPath("Symbol");
-            var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", path, false);
+            var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, path);
             Assert.True(task.Log.HasLoggedErrors);
         }
 
@@ -94,9 +94,9 @@ public void TemporarySymbolsDirectoryTest()
         {
             var publishTask = new PublishArtifactsInManifestV3();
             var path = TestInputs.GetFullPath("Test");
-            publishTask.EnsureTemporarySymbolDirectoryExists(path);
+            publishTask.EnsureTemporaryDirectoryExists(path);
             Assert.True(Directory.Exists(path));
-            publishTask.DeleteSymbolTemporaryDirectory(path);
+            publishTask.DeleteTemporaryDirectory(path);
             Assert.False(Directory.Exists(path));
         }
 

src/Microsoft.DotNet.Build.Tasks.Feed/src/BuildArtifacts.cs

@@ -0,0 +1,35 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Microsoft.DotNet.Build.Tasks.Feed
+{
+    public class Properties
+    {
+        public string artifactsize { get; set; }
+        public string RootId { get; set; }
+        public string localpath { get; set; }
+    }
+
+    public class Resource
+    {
+        public string type { get; set; }
+        public string data { get; set; }
+        public Properties properties { get; set; }
+        public string url { get; set; }
+        public string downloadUrl { get; set; }
+    }
+
+    public class Value
+    {
+        public int id { get; set; }
+        public string name { get; set; }
+        public string source { get; set; }
+        public Resource resource { get; set; }
+    }
+
+    public class BuildArtifacts
+    {
+        public int count { get; set; }
+        public IList<Value> value { get; set; }
+    }
+}

src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs

@@ -183,6 +183,17 @@ public string BuildQuality
             get { return _buildQuality.GetDescription(); }
             set { Enum.TryParse<PublishingConstants.BuildQuality>(value, true, out _buildQuality); }
         }
+        public string AzdoApiToken {get; set;}
+
+        public string StagingDir { get; set;}
+
+        public string BuildId { get; set; }
+
+        public string AzureProject { get; set; }
+
+        public string AzureDevOpsOrg { get; set; }
+
+        public bool UseApiOverride {get; set;}
 
         /// <summary>
         /// Just an internal flag to keep track whether we published assets via a V3 manifest or not.
@@ -372,7 +383,13 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui
                 ChecksumsFeedOverride = this.ChecksumsFeedOverride,
                 ShippingFeedOverride = this.ShippingFeedOverride,
                 TransportFeedOverride = this.TransportFeedOverride,
-                SymbolsFeedOverride = this.SymbolsFeedOverride
+                SymbolsFeedOverride = this.SymbolsFeedOverride,
+                StagingDir =  this.StagingDir,
+                AzdoApiToken = this.AzdoApiToken,
+                BuildId = this.BuildId,
+                AzureProject = this.AzureProject,
+                AzureDevOpsOrg = this.AzureDevOpsOrg,
+                UseApiOverride = this.UseApiOverride
             };
         }
     }