Skip to content

Commit d2ab0a7

Browse files
nakulkar-msftnakulkar-msft
committed
Squashed commit of the following:
(Merge origin/dev) commit 380f113 Author: Narasimha Kulkarni <nakulkar@microsoft.com> Date: Wed Mar 30 08:05:12 2022 +0530 Fix memleak in chunkedFileWriter (#1695) * Fix memleak in chunkedFileWriter * Ensure Flush waits for workerRoutine to exit * Rename variable to a more meaningful one commit d76b7bf Author: Narasimha Kulkarni <nakulkar@microsoft.com> Date: Wed Mar 30 08:04:49 2022 +0530 Refactor STE: Part 3 (#1732) * Move global status manager to jobMgr * Initialize exculsive string map before first order * Add daemon Mode to jobMgr * Notify on completion commit ddec1b9 Author: Narasimha Kulkarni <nakulkar@microsoft.com> Date: Tue Mar 29 15:21:56 2022 +0530 Env var to control downloading to temp path (#1667) * Env var to control downloading to temp path * Implement TempPath method in mockedLCM commit 4360757 Author: Narasimha Kulkarni <nakulkar@microsoft.com> Date: Mon Mar 28 16:09:29 2022 +0530 Refactor STE: Separate JobsAdmin out of STE (#1719) commit 93a3b74 Author: Narasimha Kulkarni <nakulkar@microsoft.com> Date: Thu Mar 24 07:48:09 2022 +0530 Introduce syslogger to be used with STE (#1721) * Introduce syslogger to be used with STE * Fix windows build * Address reviews commit 832fa4c Author: Mohit Sharma <65536214+mohsha-msft@users.noreply.github.com> Date: Thu Mar 24 07:46:15 2022 +0530 Changed timeout to 60m (#1718) commit 1ac90a4 Author: adreed-msft <49764384+adreed-msft@users.noreply.github.com> Date: Wed Mar 16 11:07:56 2022 -0700 Implement OAuth in the modern test framework (#1410) * Configure Azure Pipelines for OAuth on E2E * Handle separate source/destination request lists * Reduce surface area of oauth testing * Update function signatures * Fix panic * Set OAuth info * Fix auth for remove * Fix resume support * Fix objectTarget * Fix CopyToWrongBlobType * Fix StripTopDir * Reduce scale of TestResume_LargeGeneric * Add CPK vars * Add HNS key * Fix test definitions * Add E2E classic account key * Resolve Mohit's comments commit 6e2c8e7 Author: Mohit Sharma <65536214+mohsha-msft@users.noreply.github.com> Date: Wed Mar 16 21:07:10 2022 +0530 Fix error message in local traverser (issue #849) (#1713) * Log the failure to scan local files to help debugging * return error trace as well. Co-authored-by: Adele Reed <adreed@microsoft.com> commit e2673aa Author: Ze Qian Zhang <zezha@microsoft.com> Date: Wed Mar 16 03:39:13 2022 -0700 Dev <- Main after 10.14.1 (#1712) * Fix Invalid Blob Name (#1608) * Revert "Fix Invalid Blob Name (#1608)" (#1644) This reverts commit deac65d. * Fix handling of env variable (#1686) * Added Component Governance Component Detection (#1690) Co-authored-by: Mohit Sharma <65536214+mohsha-msft@users.noreply.github.com> Co-authored-by: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> commit 84bd216 Author: Mohit Sharma <65536214+mohsha-msft@users.noreply.github.com> Date: Wed Mar 16 14:08:31 2022 +0530 MinIO TraceOn in DEBUG mode (#1678) * Added TraceOn in MinIO * Minor Edits * Redesigned * Changes when logger is nil. Passing unit tests. * Scrubbing off unnecessary changes * Removed extra changes commit 7ee54bc Author: Mohit Sharma <65536214+mohsha-msft@users.noreply.github.com> Date: Wed Mar 16 13:29:36 2022 +0530 Updated Dependencies. (#1708) commit f4e5770 Author: Ze Qian Zhang <zezha@microsoft.com> Date: Tue Mar 15 11:01:13 2022 -0700 Merge main back to dev after 10.14 (#1691) * Fix Invalid Blob Name (#1608) * Revert "Fix Invalid Blob Name (#1608)" (#1644) This reverts commit deac65d. * Fix handling of env variable (#1686) * Added Component Governance Component Detection (#1690) Co-authored-by: Mohit Sharma <65536214+mohsha-msft@users.noreply.github.com> Co-authored-by: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com>
1 parent c8cfae2 commit d2ab0a7

File tree

81 files changed

+1570
-1118
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

81 files changed

+1570
-1118
lines changed

‎azbfs/zc_filesystemRequestOptions.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,4 @@ type ListPathsFilesystemOptions struct {
1515
MaxResults *int32
1616
// The continuation token to resume listing.
1717
ContinuationToken *string
18-
}
18+
}

‎azure-pipelines.yml

+19-4
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,12 @@ jobs:
6767
- task: PublishBuildArtifacts@1
6868
displayName: 'Publish Artifacts'
6969
condition: succeededOrFailed()
70+
71+
- task: ComponentGovernanceComponentDetection@0
72+
inputs:
73+
scanType: 'Register'
74+
verbosity: 'Verbose'
75+
alertWarningLevel: 'High'
7076

7177
- job: E2E_Test
7278
timeoutInMinutes: 360
@@ -95,12 +101,15 @@ jobs:
95101
set -e
96102
GOARCH=amd64 GOOS=linux go build -o azcopy_linux_amd64
97103
export AZCOPY_E2E_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64
98-
go test -timeout 40m -race -short -cover ./e2etest
104+
go test -timeout 60m -race -short -cover ./e2etest
99105
env:
100106
AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY)
101107
AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME)
102108
AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS)
103109
AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS)
110+
AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID)
111+
AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID)
112+
AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
104113
AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
105114
AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
106115
CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
@@ -113,12 +122,15 @@ jobs:
113122
go build -o $(System.DefaultWorkingDirectory)/azcopy_windows_amd64.exe
114123
go build -o $(System.DefaultWorkingDirectory)/azcopy_windows_386.exe
115124
echo 'starting E2E tests on windows'
116-
go test -timeout 40m -race -cover -v ./e2etest
125+
go test -timeout 60m -race -cover -v ./e2etest
117126
env:
118127
AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY)
119128
AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME)
120129
AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS)
121130
AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS)
131+
AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID)
132+
AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID)
133+
AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
122134
AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
123135
AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
124136
CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
@@ -133,12 +145,15 @@ jobs:
133145
go build -o azcopy_darwin_amd64
134146
echo 'starting E2E tests on mac-os'
135147
export AZCOPY_E2E_EXECUTABLE_PATH=$(pwd)/azcopy_darwin_amd64
136-
go test -timeout 40m -race -cover -v ./e2etest
148+
go test -timeout 60m -race -cover -v ./e2etest
137149
env:
138150
AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY)
139151
AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME)
140152
AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS)
141153
AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS)
154+
AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID)
155+
AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID)
156+
AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
142157
AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
143158
AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
144159
CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
@@ -181,7 +196,7 @@ jobs:
181196
# the set -e line is needed so that the unit tests failure would cause the job to fail properly
182197
# "-check.v" (must be after package list) outputs timings
183198
set -e
184-
go test -timeout 45m -race -short -cover ./cmd ./common ./common/parallel ./ste ./azbfs ./sddl "-check.v"
199+
go test -timeout 60m -race -short -cover ./cmd ./common ./common/parallel ./ste ./azbfs ./sddl "-check.v"
185200
GOARCH=amd64 GOOS=linux go build -o azcopy_linux_amd64
186201
name: 'Run_unit_tests'
187202
env:

‎cmd/copy.go

+6-5
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"encoding/json"
2727
"errors"
2828
"fmt"
29+
"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
2930
"io"
3031
"math"
3132
"net/url"
@@ -1313,8 +1314,8 @@ func (cca *CookedCopyCmdArgs) processRedirectionUpload(blobResource common.Resou
13131314
func (cca *CookedCopyCmdArgs) processCopyJobPartOrders() (err error) {
13141315
ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion)
13151316
// Make AUTO default for Azure Files since Azure Files throttles too easily.
1316-
if ste.JobsAdmin != nil && (cca.FromTo.From() == common.ELocation.File() || cca.FromTo.To() == common.ELocation.File()) {
1317-
ste.JobsAdmin.SetConcurrencySettingsToAuto()
1317+
if jobsAdmin.JobsAdmin != nil && (cca.FromTo.From() == common.ELocation.File() || cca.FromTo.To() == common.ELocation.File()) {
1318+
jobsAdmin.JobsAdmin.SetConcurrencySettingsToAuto()
13181319
}
13191320

13201321
// Note: credential info here is only used by remove at the moment.
@@ -1585,7 +1586,7 @@ TotalBytesTransferred: %v
15851586
Final Job Status: %v%s%s
15861587
`,
15871588
summary.JobID.String(),
1588-
ste.ToFixed(duration.Minutes(), 4),
1589+
jobsAdmin.ToFixed(duration.Minutes(), 4),
15891590
summary.FileTransfers,
15901591
summary.FolderPropertyTransfers,
15911592
summary.TotalTransfers,
@@ -1603,7 +1604,7 @@ Final Job Status: %v%s%s
16031604
}
16041605

16051606
// log to job log
1606-
jobMan, exists := ste.JobsAdmin.JobMgr(summary.JobID)
1607+
jobMan, exists := jobsAdmin.JobsAdmin.JobMgr(summary.JobID)
16071608
if exists {
16081609
jobMan.Log(pipeline.LogInfo, logStats+"\n"+output)
16091610
}
@@ -1651,7 +1652,7 @@ Final Job Status: %v%s%s
16511652
}
16521653

16531654
throughput := computeThroughput()
1654-
throughputString := fmt.Sprintf("2-sec Throughput (Mb/s): %v", ste.ToFixed(throughput, 4))
1655+
throughputString := fmt.Sprintf("2-sec Throughput (Mb/s): %v", jobsAdmin.ToFixed(throughput, 4))
16551656
if throughput == 0 {
16561657
// As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it.
16571658
throughputString = ""

‎cmd/copyEnumeratorHelper.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package cmd
33
import (
44
"fmt"
55
"github.com/Azure/azure-pipeline-go/pipeline"
6-
"github.com/Azure/azure-storage-azcopy/v10/ste"
6+
"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
77
"math/rand"
88
"strings"
99

@@ -85,8 +85,8 @@ func dispatchFinalPart(e *common.CopyJobPartOrderRequest, cca *CookedCopyCmdArgs
8585
return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNum, resp.ErrorMsg)
8686
}
8787

88-
if ste.JobsAdmin != nil {
89-
ste.JobsAdmin.LogToJobLog(FinalPartCreatedMessage, pipeline.LogInfo)
88+
if jobsAdmin.JobsAdmin != nil {
89+
jobsAdmin.JobsAdmin.LogToJobLog(FinalPartCreatedMessage, pipeline.LogInfo)
9090
}
9191

9292
// set the flag on cca, to indicate the enumeration is done

‎cmd/copyEnumeratorInit.go

+12-12
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"encoding/json"
66
"errors"
77
"fmt"
8+
"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
89
"log"
910
"net/url"
1011
"os"
@@ -20,7 +21,6 @@ import (
2021
"github.com/Azure/azure-storage-file-go/azfile"
2122

2223
"github.com/Azure/azure-storage-azcopy/v10/common"
23-
"github.com/Azure/azure-storage-azcopy/v10/ste"
2424
)
2525

2626
type BucketToContainerNameResolver interface {
@@ -151,11 +151,11 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
151151
err = cca.createDstContainer(dstContainerName, cca.Destination, ctx, existingContainers, cca.LogVerbosity)
152152

153153
// check against seenFailedContainers so we don't spam the job log with initialization failed errors
154-
if _, ok := seenFailedContainers[dstContainerName]; err != nil && ste.JobsAdmin != nil && !ok {
154+
if _, ok := seenFailedContainers[dstContainerName]; err != nil && jobsAdmin.JobsAdmin != nil && !ok {
155155
logDstContainerCreateFailureOnce.Do(func() {
156156
glcm.Info("Failed to create one or more destination container(s). Your transfers may still succeed if the container already exists.")
157157
})
158-
ste.JobsAdmin.LogToJobLog(fmt.Sprintf("failed to initialize destination container %s; the transfer will continue (but be wary it may fail): %s", dstContainerName, err), pipeline.LogWarning)
158+
jobsAdmin.JobsAdmin.LogToJobLog(fmt.Sprintf("failed to initialize destination container %s; the transfer will continue (but be wary it may fail): %s", dstContainerName, err), pipeline.LogWarning)
159159
seenFailedContainers[dstContainerName] = true
160160
}
161161
} else if cca.FromTo.From().IsRemote() { // if the destination has implicit container names
@@ -187,11 +187,11 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
187187
// if JobsAdmin is nil, we're probably in testing mode.
188188
// As a result, container creation failures are expected as we don't give the SAS tokens adequate permissions.
189189
// check against seenFailedContainers so we don't spam the job log with initialization failed errors
190-
if _, ok := seenFailedContainers[bucketName]; err != nil && ste.JobsAdmin != nil && !ok {
190+
if _, ok := seenFailedContainers[bucketName]; err != nil && jobsAdmin.JobsAdmin != nil && !ok {
191191
logDstContainerCreateFailureOnce.Do(func() {
192192
glcm.Info("Failed to create one or more destination container(s). Your transfers may still succeed if the container already exists.")
193193
})
194-
ste.JobsAdmin.LogToJobLog(fmt.Sprintf("failed to initialize destination container %s; the transfer will continue (but be wary it may fail): %s", bucketName, err), pipeline.LogWarning)
194+
jobsAdmin.JobsAdmin.LogToJobLog(fmt.Sprintf("failed to initialize destination container %s; the transfer will continue (but be wary it may fail): %s", bucketName, err), pipeline.LogWarning)
195195
seenFailedContainers[bucketName] = true
196196
}
197197
}
@@ -207,11 +207,11 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
207207
if err == nil {
208208
err = cca.createDstContainer(resName, cca.Destination, ctx, existingContainers, cca.LogVerbosity)
209209

210-
if _, ok := seenFailedContainers[dstContainerName]; err != nil && ste.JobsAdmin != nil && !ok {
210+
if _, ok := seenFailedContainers[dstContainerName]; err != nil && jobsAdmin.JobsAdmin != nil && !ok {
211211
logDstContainerCreateFailureOnce.Do(func() {
212212
glcm.Info("Failed to create one or more destination container(s). Your transfers may still succeed if the container already exists.")
213213
})
214-
ste.JobsAdmin.LogToJobLog(fmt.Sprintf("failed to initialize destination container %s; the transfer will continue (but be wary it may fail): %s", dstContainerName, err), pipeline.LogWarning)
214+
jobsAdmin.JobsAdmin.LogToJobLog(fmt.Sprintf("failed to initialize destination container %s; the transfer will continue (but be wary it may fail): %s", dstContainerName, err), pipeline.LogWarning)
215215
seenFailedContainers[dstContainerName] = true
216216
}
217217
}
@@ -227,8 +227,8 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
227227
if !cca.dryrunMode {
228228
glcm.Info(message)
229229
}
230-
if ste.JobsAdmin != nil {
231-
ste.JobsAdmin.LogToJobLog(message, pipeline.LogInfo)
230+
if jobsAdmin.JobsAdmin != nil {
231+
jobsAdmin.JobsAdmin.LogToJobLog(message, pipeline.LogInfo)
232232
}
233233

234234
processor := func(object StoredObject) error {
@@ -410,9 +410,9 @@ func (cca *CookedCopyCmdArgs) InitModularFilters() []ObjectFilter {
410410
}
411411

412412
// finally, log any search prefix computed from these
413-
if ste.JobsAdmin != nil {
413+
if jobsAdmin.JobsAdmin != nil {
414414
if prefixFilter := FilterSet(filters).GetEnumerationPreFilter(cca.Recursive); prefixFilter != "" {
415-
ste.JobsAdmin.LogToJobLog("Search prefix, which may be used to optimize scanning, is: "+prefixFilter, pipeline.LogInfo) // "May be used" because we don't know here which enumerators will use it
415+
jobsAdmin.JobsAdmin.LogToJobLog("Search prefix, which may be used to optimize scanning, is: "+prefixFilter, pipeline.LogInfo) // "May be used" because we don't know here which enumerators will use it
416416
}
417417
}
418418

@@ -657,7 +657,7 @@ func (cca *CookedCopyCmdArgs) MakeEscapedRelativePath(source bool, dstIsDir bool
657657
panic("unexpected inescapable rootDir name")
658658
}
659659
}
660-
660+
661661
relativePath = "/" + rootDir + relativePath
662662
}
663663

‎cmd/copyUtil.go

+5-6
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,14 @@ package cmd
2323
import (
2424
"context"
2525
"fmt"
26+
"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
2627
"net/url"
2728
"os"
2829
"strings"
2930

3031
"github.com/Azure/azure-pipeline-go/pipeline"
3132
"github.com/Azure/azure-storage-azcopy/v10/azbfs"
3233
"github.com/Azure/azure-storage-azcopy/v10/common"
33-
"github.com/Azure/azure-storage-azcopy/v10/ste"
34-
3534
"github.com/Azure/azure-storage-blob-go/azblob"
3635
"github.com/Azure/azure-storage-file-go/azfile"
3736
)
@@ -132,8 +131,8 @@ func (util copyHandlerUtil) urlIsBFSFileSystemOrDirectory(ctx context.Context, u
132131
isDir, err := dirURL.IsDirectory(ctx)
133132

134133
if err != nil {
135-
if ste.JobsAdmin != nil {
136-
ste.JobsAdmin.LogToJobLog(fmt.Sprintf("Failed to check if destination is a folder or a file (ADLSg2). Assuming the destination is a file: %s", err), pipeline.LogWarning)
134+
if jobsAdmin.JobsAdmin != nil {
135+
jobsAdmin.JobsAdmin.LogToJobLog(fmt.Sprintf("Failed to check if destination is a folder or a file (ADLSg2). Assuming the destination is a file: %s", err), pipeline.LogWarning)
137136
}
138137
}
139138

@@ -150,8 +149,8 @@ func (util copyHandlerUtil) urlIsAzureFileDirectory(ctx context.Context, url *ur
150149
directoryURL := azfile.NewDirectoryURL(*url, p)
151150
_, err := directoryURL.GetProperties(ctx)
152151
if err != nil {
153-
if ste.JobsAdmin != nil {
154-
ste.JobsAdmin.LogToJobLog(fmt.Sprintf("Failed to check if the destination is a folder or a file (Azure Files). Assuming the destination is a file: %s", err), pipeline.LogWarning)
152+
if jobsAdmin.JobsAdmin != nil {
153+
jobsAdmin.JobsAdmin.LogToJobLog(fmt.Sprintf("Failed to check if the destination is a folder or a file (Azure Files). Assuming the destination is a file: %s", err), pipeline.LogWarning)
155154
}
156155

157156
return false

‎cmd/credentialUtil.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"context"
2727
"errors"
2828
"fmt"
29+
"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
2930
"net/http"
3031
"net/url"
3132
"strings"
@@ -460,8 +461,8 @@ func logAuthType(ct common.CredentialType, location common.Location, isSource bo
460461
message := fmt.Sprintf("Authenticating to %s using %s", resource, name)
461462
if _, exists := authMessagesAlreadyLogged.Load(message); !exists {
462463
authMessagesAlreadyLogged.Store(message, struct{}{}) // dedup because source is auth'd by both enumerator and STE
463-
if ste.JobsAdmin != nil {
464-
ste.JobsAdmin.LogToJobLog(message, pipeline.LogInfo)
464+
if jobsAdmin.JobsAdmin != nil {
465+
jobsAdmin.JobsAdmin.LogToJobLog(message, pipeline.LogInfo)
465466
}
466467
glcm.Info(message)
467468
}

‎cmd/jobsClean.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ func handleCleanJobsCommand(givenStatus common.JobStatus) error {
113113

114114
func blindDeleteAllJobFiles() (int, error) {
115115
// get rid of the job plan files
116-
numPlanFilesRemoved, err := removeFilesWithPredicate(azcopyJobPlanFolder, func(s string) bool {
116+
numPlanFilesRemoved, err := removeFilesWithPredicate(common.AzcopyJobPlanFolder, func(s string) bool {
117117
if strings.Contains(s, ".steV") {
118118
return true
119119
}

‎cmd/jobsRemove.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ func init() {
7575

7676
func handleRemoveSingleJob(jobID common.JobID) error {
7777
// get rid of the job plan files
78-
numPlanFileRemoved, err := removeFilesWithPredicate(azcopyJobPlanFolder, func(s string) bool {
78+
numPlanFileRemoved, err := removeFilesWithPredicate(common.AzcopyJobPlanFolder, func(s string) bool {
7979
if strings.Contains(s, jobID.String()) && strings.Contains(s, ".steV") {
8080
return true
8181
}

‎cmd/jobsResume.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"encoding/json"
2626
"errors"
2727
"fmt"
28+
"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
2829
"strings"
2930
"time"
3031

@@ -109,7 +110,7 @@ func (cca *resumeJobController) ReportProgressOrExit(lcm common.LifecycleMgr) (t
109110
return fmt.Sprintf(
110111
"\n\nJob %s summary\nElapsed Time (Minutes): %v\nNumber of File Transfers: %v\nNumber of Folder Property Transfers: %v\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nTotalBytesTransferred: %v\nFinal Job Status: %v\n",
111112
summary.JobID.String(),
112-
ste.ToFixed(duration.Minutes(), 4),
113+
jobsAdmin.ToFixed(duration.Minutes(), 4),
113114
summary.FileTransfers,
114115
summary.FolderPropertyTransfers,
115116
summary.TotalTransfers,
@@ -148,7 +149,7 @@ func (cca *resumeJobController) ReportProgressOrExit(lcm common.LifecycleMgr) (t
148149
}
149150

150151
throughput := computeThroughput()
151-
throughputString := fmt.Sprintf("2-sec Throughput (Mb/s): %v", ste.ToFixed(throughput, 4))
152+
throughputString := fmt.Sprintf("2-sec Throughput (Mb/s): %v", jobsAdmin.ToFixed(throughput, 4))
152153
if throughput == 0 {
153154
// As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it.
154155
throughputString = ""

‎cmd/loadOutputTranslation.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,10 @@ import (
2424
"bufio"
2525
"encoding/json"
2626
"fmt"
27+
"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
2728
"io"
2829
"strings"
2930

30-
"github.com/Azure/azure-storage-azcopy/v10/ste"
31-
3231
"github.com/Azure/azure-storage-azcopy/v10/common"
3332
)
3433

@@ -225,7 +224,7 @@ Number of Transfers Skipped: %v
225224
TotalBytesTransferred: %v
226225
Final Job Status: %v
227226
`,
228-
ste.ToFixed(summary.Elapsed/60, 4),
227+
jobsAdmin.ToFixed(summary.Elapsed/60, 4),
229228
summary.FilesTotal,
230229
summary.FilesCompleted,
231230
summary.FilesFailed,

‎cmd/removeEnumerator.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"encoding/json"
2626
"errors"
2727
"fmt"
28+
"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
2829
"strings"
2930

3031
"github.com/Azure/azure-pipeline-go/pipeline"
@@ -75,8 +76,8 @@ func newRemoveEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator, er
7576
if !cca.dryrunMode {
7677
glcm.Info(message)
7778
}
78-
if ste.JobsAdmin != nil {
79-
ste.JobsAdmin.LogToJobLog(message, pipeline.LogInfo)
79+
if jobsAdmin.JobsAdmin != nil {
80+
jobsAdmin.JobsAdmin.LogToJobLog(message, pipeline.LogInfo)
8081
}
8182

8283
transferScheduler := newRemoveTransferProcessor(cca, NumOfFilesPerDispatchJobPart, fpo)

0 commit comments

Comments
 (0)