Skip to content

Commit 84bd216

Browse files
mohsha-msftmohsha-msft
authored
MinIO TraceOn in DEBUG mode (#1678)
* Added TraceOn in MinIO * Minor Edits * Redesigned * Changes when logger is nil. Passing unit tests. * Scrubbing off unnecessary changes * Removed extra changes
1 parent 7ee54bc commit 84bd216

5 files changed

+59
-44
lines changed

‎cmd/zc_traverser_s3.go

+12-14
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@ func (t *s3Traverser) Traverse(preprocessor objectMorpher, processor objectProce
7979
objectName := objectPath[len(objectPath)-1]
8080

8181
oi, err := t.s3Client.StatObject(t.s3URLParts.BucketName, t.s3URLParts.ObjectKey, minio.StatObjectOptions{})
82-
8382
if invalidAzureBlobName(t.s3URLParts.ObjectKey) {
8483
WarnStdoutAndScanningLog(fmt.Sprintf(invalidNameErrorMsg, t.s3URLParts.ObjectKey))
8584
return common.EAzError.InvalidBlobName()
@@ -186,8 +185,10 @@ func (t *s3Traverser) Traverse(preprocessor objectMorpher, processor objectProce
186185
return
187186
}
188187

189-
func newS3Traverser(credentialType common.CredentialType, rawURL *url.URL, ctx context.Context, recursive, getProperties bool, incrementEnumerationCounter enumerationCounterFunc) (t *s3Traverser, err error) {
190-
t = &s3Traverser{rawURL: rawURL, ctx: ctx, recursive: recursive, getProperties: getProperties, incrementEnumerationCounter: incrementEnumerationCounter}
188+
func newS3Traverser(credentialType common.CredentialType, rawURL *url.URL, ctx context.Context, recursive, getProperties bool,
189+
incrementEnumerationCounter enumerationCounterFunc) (t *s3Traverser, err error) {
190+
t = &s3Traverser{rawURL: rawURL, ctx: ctx, recursive: recursive, getProperties: getProperties,
191+
incrementEnumerationCounter: incrementEnumerationCounter}
191192

192193
// initialize S3 client and URL parts
193194
var s3URLParts common.S3URLParts
@@ -201,18 +202,15 @@ func newS3Traverser(credentialType common.CredentialType, rawURL *url.URL, ctx c
201202

202203
showS3UrlTypeWarning(s3URLParts)
203204

204-
t.s3Client, err = common.CreateS3Client(
205-
t.ctx,
206-
common.CredentialInfo{
207-
CredentialType: credentialType,
208-
S3CredentialInfo: common.S3CredentialInfo{
209-
Endpoint: t.s3URLParts.Endpoint,
210-
Region: t.s3URLParts.Region,
211-
},
205+
t.s3Client, err = common.CreateS3Client(t.ctx, common.CredentialInfo{
206+
CredentialType: credentialType,
207+
S3CredentialInfo: common.S3CredentialInfo{
208+
Endpoint: t.s3URLParts.Endpoint,
209+
Region: t.s3URLParts.Region,
212210
},
213-
common.CredentialOpOptions{
214-
LogError: glcm.Error,
215-
})
211+
}, common.CredentialOpOptions{
212+
LogError: glcm.Error,
213+
}, azcopyScanningLogger)
216214

217215
return
218216
}

‎cmd/zc_traverser_s3_service.go

+10-13
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,8 @@ func (t *s3ServiceTraverser) IsDirectory(isSource bool) bool {
5555
func (t *s3ServiceTraverser) listContainers() ([]string, error) {
5656
if len(t.cachedBuckets) == 0 {
5757
bucketList := make([]string, 0)
58-
59-
if bucketInfo, err := t.s3Client.ListBuckets(); err == nil {
58+
bucketInfo, err := t.s3Client.ListBuckets()
59+
if err == nil {
6060
for _, v := range bucketInfo {
6161
// Match a pattern for the bucket name and the bucket name only
6262
if t.bucketPattern != "" {
@@ -120,6 +120,7 @@ func (t *s3ServiceTraverser) Traverse(preprocessor objectMorpher, processor obje
120120
}
121121
}
122122

123+
t.s3Client.TraceOff()
123124
return nil
124125
}
125126

@@ -142,17 +143,13 @@ func newS3ServiceTraverser(rawURL *url.URL, ctx context.Context, getProperties b
142143

143144
t.s3URL = s3URLPartsExtension{s3URLParts}
144145

145-
t.s3Client, err = common.CreateS3Client(
146-
t.ctx,
147-
common.CredentialInfo{
148-
CredentialType: common.ECredentialType.S3AccessKey(),
149-
S3CredentialInfo: common.S3CredentialInfo{
150-
Endpoint: t.s3URL.Endpoint,
151-
},
146+
t.s3Client, err = common.CreateS3Client(t.ctx, common.CredentialInfo{
147+
CredentialType: common.ECredentialType.S3AccessKey(),
148+
S3CredentialInfo: common.S3CredentialInfo{
149+
Endpoint: t.s3URL.Endpoint,
152150
},
153-
common.CredentialOpOptions{
154-
LogError: glcm.Error,
155-
})
156-
151+
}, common.CredentialOpOptions{
152+
LogError: glcm.Error,
153+
}, azcopyScanningLogger)
157154
return
158155
}

‎common/credentialFactory.go

+9-4
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"context"
2626
"errors"
2727
"fmt"
28+
"github.com/Azure/azure-pipeline-go/pipeline"
2829
"math"
2930
"strings"
3031
"sync"
@@ -237,7 +238,7 @@ func refreshBlobFSToken(ctx context.Context, tokenInfo OAuthTokenInfo, tokenCred
237238
// ==============================================================================================
238239
// S3 credential related factory methods
239240
// ==============================================================================================
240-
func CreateS3Client(ctx context.Context, credInfo CredentialInfo, option CredentialOpOptions) (*minio.Client, error) {
241+
func CreateS3Client(ctx context.Context, credInfo CredentialInfo, option CredentialOpOptions, logger ILogger) (*minio.Client, error) {
241242
if credInfo.CredentialType == ECredentialType.S3PublicBucket() {
242243
cred := credentials.NewStatic("", "", "", credentials.SignatureAnonymous)
243244
return minio.NewWithOptions(credInfo.S3CredentialInfo.Endpoint, &minio.Options{Creds: cred, Secure: true, Region: credInfo.S3CredentialInfo.Region})
@@ -247,8 +248,12 @@ func CreateS3Client(ctx context.Context, credInfo CredentialInfo, option Credent
247248
if err != nil {
248249
return nil, err
249250
}
251+
s3Client, err := minio.NewWithCredentials(credInfo.S3CredentialInfo.Endpoint, credential, true, credInfo.S3CredentialInfo.Region)
250252

251-
return minio.NewWithCredentials(credInfo.S3CredentialInfo.Endpoint, credential, true, credInfo.S3CredentialInfo.Region)
253+
if logger != nil {
254+
s3Client.TraceOn(NewS3HTTPTraceLogger(logger, pipeline.LogDebug))
255+
}
256+
return s3Client, err
252257
}
253258

254259
type S3ClientFactory struct {
@@ -264,7 +269,7 @@ func NewS3ClientFactory() S3ClientFactory {
264269
}
265270

266271
// GetS3Client gets S3 client from pool, or create a new S3 client if no client created for specific credInfo.
267-
func (f *S3ClientFactory) GetS3Client(ctx context.Context, credInfo CredentialInfo, option CredentialOpOptions) (*minio.Client, error) {
272+
func (f *S3ClientFactory) GetS3Client(ctx context.Context, credInfo CredentialInfo, option CredentialOpOptions, logger ILogger) (*minio.Client, error) {
268273
f.lock.RLock()
269274
s3Client, ok := f.s3Clients[credInfo]
270275
f.lock.RUnlock()
@@ -276,7 +281,7 @@ func (f *S3ClientFactory) GetS3Client(ctx context.Context, credInfo CredentialIn
276281
f.lock.Lock()
277282
defer f.lock.Unlock()
278283
if s3Client, ok := f.s3Clients[credInfo]; !ok {
279-
newS3Client, err := CreateS3Client(ctx, credInfo, option)
284+
newS3Client, err := CreateS3Client(ctx, credInfo, option, logger)
280285
if err != nil {
281286
return nil, err
282287
}

‎common/logger.go

+18
Original file line numberDiff line numberDiff line change
@@ -223,3 +223,21 @@ func NewReadLogFunc(logger ILogger, fullUrl *url.URL) func(int, error, int64, in
223223
func IsForceLoggingDisabled() bool {
224224
return GetLifecycleMgr().IsForceLoggingDisabled()
225225
}
226+
227+
type S3HTTPTraceLogger struct {
228+
logger ILogger
229+
logLevel pipeline.LogLevel
230+
}
231+
232+
func NewS3HTTPTraceLogger(logger ILogger, level pipeline.LogLevel) S3HTTPTraceLogger {
233+
return S3HTTPTraceLogger{
234+
logger: logger,
235+
logLevel: level,
236+
}
237+
}
238+
239+
func (e S3HTTPTraceLogger) Write(msg []byte) (n int, err error) {
240+
toPrint := string(msg)
241+
e.logger.Log(e.logLevel, toPrint)
242+
return len(toPrint), nil
243+
}

‎ste/sourceInfoProvider-S3.go

+10-13
Original file line numberDiff line numberDiff line change
@@ -68,20 +68,17 @@ func newS3SourceInfoProvider(jptm IJobPartTransferMgr) (ISourceInfoProvider, err
6868
} else {
6969
p.credType = common.ECredentialType.S3AccessKey()
7070
}
71-
p.s3Client, err = s3ClientFactory.GetS3Client(
72-
p.jptm.Context(),
73-
common.CredentialInfo{
74-
CredentialType: p.credType,
75-
S3CredentialInfo: common.S3CredentialInfo{
76-
Endpoint: p.s3URLPart.Endpoint,
77-
Region: p.s3URLPart.Region,
78-
},
71+
p.s3Client, err = s3ClientFactory.GetS3Client(p.jptm.Context(), common.CredentialInfo{
72+
CredentialType: p.credType,
73+
S3CredentialInfo: common.S3CredentialInfo{
74+
Endpoint: p.s3URLPart.Endpoint,
75+
Region: p.s3URLPart.Region,
7976
},
80-
common.CredentialOpOptions{
81-
LogInfo: func(str string) { p.jptm.Log(pipeline.LogInfo, str) },
82-
LogError: func(str string) { p.jptm.Log(pipeline.LogError, str) },
83-
Panic: func(err error) { panic(err) },
84-
})
77+
}, common.CredentialOpOptions{
78+
LogInfo: func(str string) { p.jptm.Log(pipeline.LogInfo, str) },
79+
LogError: func(str string) { p.jptm.Log(pipeline.LogError, str) },
80+
Panic: func(err error) { panic(err) },
81+
}, jptm)
8582
if err != nil {
8683
return nil, err
8784
}

0 commit comments

Comments
 (0)