Azure
diff --git a/‎.github/CODEOWNERS
+1 b/‎.github/CODEOWNERS
+1
diff --git a/‎.github/workflows/codeql-analysis.yml
+67 b/‎.github/workflows/codeql-analysis.yml
+67
diff --git a/‎.github/workflows/codespell.yml
+24 b/‎.github/workflows/codespell.yml
+24
diff --git a/‎.github/workflows/trivy.yml
+54 b/‎.github/workflows/trivy.yml
+54
diff --git a/‎ChangeLog.md
+22-9 b/‎ChangeLog.md
+22-9
diff --git a/‎azbfs/parsing_urls.go
+1-1 b/‎azbfs/parsing_urls.go
+1-1
diff --git a/‎azbfs/zc_credential_token.go
+1-1 b/‎azbfs/zc_credential_token.go
+1-1
diff --git a/‎azure-pipelines.yml
+27-8 b/‎azure-pipelines.yml
+27-8
diff --git a/‎cmd/benchmark.go
+2-2 b/‎cmd/benchmark.go
+2-2
@@ -0,0 +1 @@
+*       @gapra-msft @adreed-msft @nakulkar-msft @siminsavani-msft @vibhansa-msft @tasherif-msft
@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  pull_request:
+    branches: [ main, dev ]
+  push:
+    branches: [ main, dev ]
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
+
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
+
+      #- run: |
+      #   make bootstrap
+      #   make release
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
@@ -0,0 +1,24 @@
+# GitHub Action to automate the identification of common misspellings in text files.
+# https://github.com/codespell-project/actions-codespell
+# https://github.com/codespell-project/codespell
+name: codespell
+on:
+  push:
+    branches:
+      - dev
+      - main
+  pull_request:
+    branches:
+      - dev
+      - main
+jobs:
+  codespell:
+    name: Check for spelling errors
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: codespell-project/actions-codespell@master
+        with:
+          check_filenames: true
+          skip: ./sddl/sddlPortable_test.go,./sddl/sddlHelper_linux.go
+          ignore_words_list: "resue,pase,cancl,cacl,froms"
@@ -0,0 +1,54 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: trivy
+
+on:
+  push:
+    branches: [ "main", "dev" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "main", "dev" ]
+  schedule:
+    - cron: '31 19 * * 1'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
+
+    name: Build
+    runs-on: "ubuntu-22.04"
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Build AzCopy
+        run: |
+          go build -o azcopy
+          ls -l
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: fs
+          scan-ref: './azcopy'
+          ignore-unfixed: true
+          format: 'sarif'
+          output: 'trivy-results-binary.sarif'
+          severity: 'CRITICAL,HIGH,MEDIUM,LOW'
+
+      - name: List Issues
+        run: |
+          cat trivy-results-binary.sarif
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: 'trivy-results-binary.sarif'
@@ -1,6 +1,19 @@
 
 # Change Log
 
+## Version 10.17.0
+
+### New features
+
+1. Added support for hash-based sync. AzCopy sync can now take two new flags `--compare-hash` and `--missing-hash-policy=Generate`, which which user will be able to transfer only those files which differ in their MD5 hash.
+
+### Bug fixes
+1. Fixed [issue 1994](https://github.com/Azure/azure-storage-azcopy/pull/1994): Error in calculation of block size
+2. Fixed [issue 1957](https://github.com/Azure/azure-storage-azcopy/pull/1957): Repeated Authentication token refresh
+3. Fixed [issue 1870](https://github.com/Azure/azure-storage-azcopy/pull/1870): Fixed issue where CPK would not be injected on retries
+4. Fixed [issue 1946](https://github.com/Azure/azure-storage-azcopy/issues/1946): Fixed Metadata parsing
+5: Fixed [issue 1931](https://github.com/Azure/azure-storage-azcopy/issues/1931)
+
 ## Version 10.16.2
 
 ### Bug Fixes
@@ -35,7 +48,7 @@
 1. Fixed [issue 1506](https://github.com/Azure/azure-storage-azcopy/issues/1506): Added input watcher to resolve issue since job could not be resumed.
 2. Fixed [issue 1794](https://github.com/Azure/azure-storage-azcopy/issues/1794): Moved log-level to root.go so log-level arguments do not get ignored.
 3. Fixed [issue 1824](https://github.com/Azure/azure-storage-azcopy/issues/1824): Avoid creating .azcopy under HOME if plan/log location is specified elsewhere.
-4. Fixed [isue 1830](https://github.com/Azure/azure-storage-azcopy/issues/1830), [issue 1412](https://github.com/Azure/azure-storage-azcopy/issues/1418), and [issue 873](https://github.com/Azure/azure-storage-azcopy/issues/873): Improved error message for when AzCopy cannot determine if source is directory.
+4. Fixed [issue 1830](https://github.com/Azure/azure-storage-azcopy/issues/1830), [issue 1412](https://github.com/Azure/azure-storage-azcopy/issues/1418), and [issue 873](https://github.com/Azure/azure-storage-azcopy/issues/873): Improved error message for when AzCopy cannot determine if source is directory.
 5. Fixed [issue 1777](https://github.com/Azure/azure-storage-azcopy/issues/1777): Fixed job list to handle respective output-type correctly. 
 6. Fixed win64 alignment issue.
 
@@ -191,7 +204,7 @@
 
 ### New features
 1. Added option to [disable parallel blob listing](https://github.com/Azure/azure-storage-azcopy/pull/1263)
-1. Added support for uploading [large files](https://github.com/Azure/azure-storage-azcopy/pull/1254/files) upto 4TiB. Please refer the [public documentation](https://docs.microsoft.com/en-us/rest/api/storageservices/create-file) for more information
+1. Added support for uploading [large files](https://github.com/Azure/azure-storage-azcopy/pull/1254/files) up to 4TiB. Please refer the [public documentation](https://docs.microsoft.com/en-us/rest/api/storageservices/create-file) for more information
 1. Added support for `include-before`flag. Refer [this](https://github.com/Azure/azure-storage-azcopy/issues/1075) for more information
 
 ### Bug fixes
@@ -469,7 +482,7 @@ disallowed because none (other than include-path) are respected.
 
 1. The `*` character is no longer supported as a wildcard in URLs, except for the two exceptions
    noted below. It remains supported in local file paths.
-   1. The first execption is that `/*` is still allowed at the very end of the "path" section of a
+   1. The first exception is that `/*` is still allowed at the very end of the "path" section of a
       URL. This is illustrated by the difference between these two source URLs:
       `https://account/container/virtual?SAS` and 
       `https://account/container/virtualDir/*?SAS`.  The former copies the virtual directory
@@ -501,7 +514,7 @@ disallowed because none (other than include-path) are respected.
 1. Percent complete is displayed as each job runs.
 1. VHD files are auto-detected as page blobs.
 1. A new benchmark mode allows quick and easy performance benchmarking of your network connection to
-   Blob Storage. Run AzCopy with the paramaters `bench --help` for details.  This feature is in
+   Blob Storage. Run AzCopy with the parameters `bench --help` for details.  This feature is in
    Preview status.
 1. The location for AzCopy's "plan" files can be specified with the environment variable
    `AZCOPY_JOB_PLAN_LOCATION`. (If you move the plan files and also move the log files using the existing
@@ -520,7 +533,7 @@ disallowed because none (other than include-path) are respected.
 1. Memory usage can be controlled by setting the new environment variable `AZCOPY_BUFFER_GB`.
    Decimal values are supported. Actual usage will be the value specified, plus some overhead. 
 1. An extra integrity check has been added: the length of the
-   completed desination file is checked against that of the source.
+   completed destination file is checked against that of the source.
 1. When downloading, AzCopy can automatically decompress blobs (or Azure Files) that have a
    `Content-Encoding` of `gzip` or `deflate`. To enable this behaviour, supply the `--decompress`
    parameter.
@@ -685,21 +698,21 @@ information, including those needed to set the new headers.
 
 1. For creating MD5 hashes when uploading, version 10.x now has the OPPOSITE default to version
    AzCopy 8.x. Specifically, as of version 10.0.9, MD5 hashes are NOT created by default. To create
-   Content-MD5 hashs when uploading, you must now specify `--put-md5` on the command line.
+   Content-MD5 hashes when uploading, you must now specify `--put-md5` on the command line.
 
 ### New features
 
 1. Can migrate data directly from Amazon Web Services (AWS). In this high-performance data path
    the data is read directly from AWS by the Azure Storage service. It does not need to pass through
-   the machine running AzCopy. The copy happens syncronously, so you can see its exact progress.  
+   the machine running AzCopy. The copy happens synchronously, so you can see its exact progress.  
 1. Can migrate data directly from Azure Files or Azure Blobs (any blob type) to Azure Blobs (any
    blob type). In this high-performance data path the data is read directly from the source by the
    Azure Storage service. It does not need to pass through the machine running AzCopy. The copy
-   happens syncronously, so you can see its exact progress.  
+   happens synchronously, so you can see its exact progress.  
 1. Sync command prompts with 4 options about deleting unneeded files from the target: Yes, No, All or
    None. (Deletion only happens if the `--delete-destination` flag is specified).
 1. Can download to /dev/null. This throws the data away - but is useful for testing raw network
-   performance unconstrained by disk; and also for validing MD5 hashes in bulk (when run in a cloud
+   performance unconstrained by disk; and also for validating MD5 hashes in bulk (when run in a cloud
    VM in the same region as the Storage account)
 
 ### Bug fixes
 
@@ -20,7 +20,7 @@ type BfsURLParts struct {
 	isIPEndpointStyle bool   // Ex: "https://ip/accountname/filesystem"
 }
 
-// isIPEndpointStyle checkes if URL's host is IP, in this case the storage account endpoint will be composed as:
+// isIPEndpointStyle checks if URL's host is IP, in this case the storage account endpoint will be composed as:
 // http(s)://IP(:port)/storageaccount/share(||container||etc)/...
 func isIPEndpointStyle(url url.URL) bool {
 	return net.ParseIP(url.Host) != nil
 
@@ -25,7 +25,7 @@ type TokenCredential interface {
 // indicating how long the TokenCredential object should wait before calling your tokenRefresher function again.
 func NewTokenCredential(initialToken string, tokenRefresher func(credential TokenCredential) time.Duration) TokenCredential {
 	tc := &tokenCredential{}
-	tc.SetToken(initialToken) // We dont' set it above to guarantee atomicity
+	tc.SetToken(initialToken) // We don't set it above to guarantee atomicity
 	if tokenRefresher == nil {
 		return tc // If no callback specified, return the simple tokenCredential
 	}
 
@@ -29,10 +29,10 @@ jobs:
         env:
           GO111MODULE: 'on'
         inputs:
-          version: '1.17.9'
+          version: '1.19.2'
 
       - script: |
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.43.0
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.46.2
           echo 'Installation complete' 
           ./bin/golangci-lint --version 
           ./bin/golangci-lint run e2etest
@@ -83,7 +83,12 @@ jobs:
 
       - script: |
           go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64"
-        displayName: 'Generate MacOS Build'
+        displayName: 'Generate MacOS Build with AMD64'
+        condition: eq(variables.type, 'mac-os')
+
+      - script: |
+          GOARCH=arm64 CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64"
+        displayName: 'Generate MacOS Build with ARM64'
         condition: eq(variables.type, 'mac-os')
 
       - task: PublishBuildArtifacts@1
@@ -116,7 +121,7 @@ jobs:
     steps:
       - task: GoTool@0
         inputs:
-          version: '1.17.9'
+          version: '1.19.2'
 
       # Running E2E Tests on Linux - AMD64
       - script: |
@@ -134,6 +139,7 @@ jobs:
           AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
           AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
           AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
+          AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs'
           AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG)
           AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG)
           CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
@@ -157,6 +163,7 @@ jobs:
           AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
           AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
           AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
+          AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs'
           AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG)
           AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG)
           CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
@@ -182,13 +189,21 @@ jobs:
           AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
           AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
           AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
+          AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs'
           AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG)
           AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG)
           CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
           CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256)
-        displayName: 'E2E Test MacOs'
+        displayName: 'E2E Test MacOs AMD64'
         condition: eq(variables.type, 'mac-os')
 
+      - task: PublishBuildArtifacts@1
+        displayName: 'Publish logs'
+        condition: succeededOrFailed()
+        inputs:
+          pathToPublish: '$(System.DefaultWorkingDirectory)/logs'
+          artifactName: logs
+
   - job: Test_On_Ubuntu
     variables:
       isMutexSet: 'false'
@@ -204,18 +219,22 @@ jobs:
       - task: GoTool@0
         name: 'Set_up_Golang'
         inputs:
-          version: '1.17.9'
+          version: '1.19.2'
       - task: DownloadSecureFile@1
         name: ciGCSServiceAccountKey
         displayName: 'Download GCS Service Account Key'
         inputs:
           secureFile: 'ci-gcs-dev.json'
       - script: |
           pip install azure-storage-blob==12.12.0
+          # set the variable to indicate that the mutex is being acquired
+          # note: we set it before acquiring the mutex to ensure we release the mutex. 
+          # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the 
+          # mutex but before we set this variable.
+          # setting this before will always work since it is valid to break an un-acquired mutex.
+          echo '##vso[task.setvariable variable=isMutexSet]true'
           # acquire the mutex before running live tests to avoid conflicts
           python ./tool_distributed_mutex.py lock "$(MUTEX_URL)"
-          # set the variable to indicate that the mutex was actually acquired
-          echo '##vso[task.setvariable variable=isMutexSet]true'
         name: 'Acquire_the_distributed_mutex'
       - script: |
           # run unit test and build executable
 
@@ -273,15 +273,15 @@ func (h benchmarkSourceHelper) FromUrl(s string) (fileCount uint, bytesPerFile i
 	pieces[0] = strings.Split(pieces[0], "=")[1]
 	pieces[1] = strings.Split(pieces[1], "=")[1]
 	pieces[2] = strings.Split(pieces[2], "=")[1]
-	fc, err := strconv.ParseUint(pieces[0], 10, 64)
+	fc, err := strconv.ParseUint(pieces[0], 10, 32)
 	if err != nil {
 		return 0, 0, 0, err
 	}
 	bpf, err := strconv.ParseInt(pieces[1], 10, 64)
 	if err != nil {
 		return 0, 0, 0, err
 	}
-	nf, err := strconv.ParseUint(pieces[2], 10, 64)
+	nf, err := strconv.ParseUint(pieces[2], 10, 32)
 	if err != nil {
 		return 0, 0, 0, err
 	}
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+* @gapra-msft @adreed-msft @nakulkar-msft @siminsavani-msft @vibhansa-msft @tasherif-msft`
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,7 @@ type BfsURLParts struct {`
`20`	`20`	`isIPEndpointStyle bool // Ex: "https://ip/accountname/filesystem"`
`21`	`21`	`}`
`22`	`22`
`23`		`-// isIPEndpointStyle checkes if URL's host is IP, in this case the storage account endpoint will be composed as:`
	`23`	`+// isIPEndpointStyle checks if URL's host is IP, in this case the storage account endpoint will be composed as:`
`24`	`24`	`// http(s)://IP(:port)/storageaccount/share(\|\|container\|\|etc)/...`
`25`	`25`	`func isIPEndpointStyle(url url.URL) bool {`
`26`	`26`	`return net.ParseIP(url.Host) != nil`
Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,7 @@ type TokenCredential interface {`
`25`	`25`	`// indicating how long the TokenCredential object should wait before calling your tokenRefresher function again.`
`26`	`26`	`func NewTokenCredential(initialToken string, tokenRefresher func(credential TokenCredential) time.Duration) TokenCredential {`
`27`	`27`	`tc := &tokenCredential{}`
`28`		`- tc.SetToken(initialToken) // We dont' set it above to guarantee atomicity`
	`28`	`+ tc.SetToken(initialToken) // We don't set it above to guarantee atomicity`
`29`	`29`	`if tokenRefresher == nil {`
`30`	`30`	`return tc // If no callback specified, return the simple tokenCredential`
`31`	`31`	`}`
Original file line number	Diff line number	Diff line change
`@@ -273,15 +273,15 @@ func (h benchmarkSourceHelper) FromUrl(s string) (fileCount uint, bytesPerFile i`
`273`	`273`	`pieces[0] = strings.Split(pieces[0], "=")[1]`
`274`	`274`	`pieces[1] = strings.Split(pieces[1], "=")[1]`
`275`	`275`	`pieces[2] = strings.Split(pieces[2], "=")[1]`
`276`		`- fc, err := strconv.ParseUint(pieces[0], 10, 64)`
	`276`	`+ fc, err := strconv.ParseUint(pieces[0], 10, 32)`
`277`	`277`	`if err != nil {`
`278`	`278`	`return 0, 0, 0, err`
`279`	`279`	`}`
`280`	`280`	`bpf, err := strconv.ParseInt(pieces[1], 10, 64)`
`281`	`281`	`if err != nil {`
`282`	`282`	`return 0, 0, 0, err`
`283`	`283`	`}`
`284`		`- nf, err := strconv.ParseUint(pieces[2], 10, 64)`
	`284`	`+ nf, err := strconv.ParseUint(pieces[2], 10, 32)`
`285`	`285`	`if err != nil {`
`286`	`286`	`return 0, 0, 0, err`
`287`	`287`	`}`