CRC: Always Structured Message

sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs

@@ -1247,18 +1247,21 @@ internal async Task<Response<BlobAppendInfo>> AppendBlockInternal(
                     string structuredBodyType = null;
                     if (validationOptions != null &&
                         validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 &&
-                        validationOptions.PrecalculatedChecksum.IsEmpty &&
                         ClientSideEncryption == null) // don't allow feature combination
                     {
                         // report progress in terms of caller bytes, not encoded bytes
                         structuredContentLength = contentLength;
                         contentLength = (content?.Length - content?.Position) ?? 0;
                         structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage;
                         content = content.WithNoDispose().WithProgress(progressHandler);
-                        content = new StructuredMessageEncodingStream(
-                            content,
-                            Constants.StructuredMessage.DefaultSegmentContentLength,
-                            StructuredMessage.Flags.StorageCrc64);
+                        content = validationOptions.PrecalculatedChecksum.IsEmpty
+                            ? new StructuredMessageEncodingStream(
+                                content,
+                                Constants.StructuredMessage.DefaultSegmentContentLength,
+                                StructuredMessage.Flags.StorageCrc64)
+                            : new StructuredMessagePrecalculatedCrcWrapperStream(
+                                content,
+                                validationOptions.PrecalculatedChecksum.Span);
                         contentLength = (content?.Length - content?.Position) ?? 0;
                     }
                     else

sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj

@@ -93,6 +93,7 @@
     <Compile Include="$(AzureStorageSharedSources)StructuredMessageDecodingRetriableStream.cs" LinkBase="Shared" />
     <Compile Include="$(AzureStorageSharedSources)StructuredMessageDecodingStream.cs" LinkBase="Shared" />
     <Compile Include="$(AzureStorageSharedSources)StructuredMessageEncodingStream.cs" LinkBase="Shared" />
+    <Compile Include="$(AzureStorageSharedSources)StructuredMessagePrecalculatedCrcWrapperStream.cs" LinkBase="Shared" />
     <Compile Include="$(AzureStorageSharedSources)UriExtensions.cs" LinkBase="Shared" />
     <Compile Include="$(AzureStorageSharedSources)UriQueryParamsCollection.cs" LinkBase="Shared" />
     <Compile Include="$(AzureStorageSharedSources)UserDelegationKeyProperties.cs" LinkBase="Shared" />

sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs

@@ -1570,6 +1570,7 @@
                                 .EnsureCompleted(),
                             async startOffset => await StructuredMessageFactory(startOffset, async: true, cancellationToken)
                                 .ConfigureAwait(false),
+                            decodedData => response.Value.Details.ContentCrc = decodedData.TotalCrc.ToArray(),
                             ClientConfiguration.Pipeline.ResponseClassifier,
                             Constants.MaxReliabilityRetries);
                     }
@@ -1719,14 +1720,7 @@
                     rangeGetContentMD5 = true;
                     break;
                 case StorageChecksumAlgorithm.StorageCrc64:
-                    if (!forceStructuredMessage && pageRange?.Length <= Constants.StructuredMessage.MaxDownloadCrcWithHeader)
-                    {
-                        rangeGetContentCRC64 = true;
-                    }
-                    else
-                    {
-                        structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage;
-                    }
+                    structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage;
                     break;
                 default:
                     break;

sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs

@@ -1337,18 +1337,21 @@ internal virtual async Task<Response<BlockInfo>> StageBlockInternal(
                     string structuredBodyType = null;
                     if (validationOptions != null &&
                         validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 &&
-                        validationOptions.PrecalculatedChecksum.IsEmpty &&
                         ClientSideEncryption == null) // don't allow feature combination
                     {
                         // report progress in terms of caller bytes, not encoded bytes
                         structuredContentLength = contentLength;
                         contentLength = (content?.Length - content?.Position) ?? 0;
                         structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage;
                         content = content.WithNoDispose().WithProgress(progressHandler);
-                        content = new StructuredMessageEncodingStream(
-                            content,
-                            Constants.StructuredMessage.DefaultSegmentContentLength,
-                            StructuredMessage.Flags.StorageCrc64);
+                        content = validationOptions.PrecalculatedChecksum.IsEmpty
+                            ? new StructuredMessageEncodingStream(
+                                content,
+                                Constants.StructuredMessage.DefaultSegmentContentLength,
+                                StructuredMessage.Flags.StorageCrc64)
+                            : new StructuredMessagePrecalculatedCrcWrapperStream(
+                                content,
+                                validationOptions.PrecalculatedChecksum.Span);
                         contentLength = (content?.Length - content?.Position) ?? 0;
                     }
                     else

sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs

@@ -34,6 +34,15 @@ public class BlobDownloadDetails
         public byte[] ContentHash { get; internal set; }
 #pragma warning restore CA1819 // Properties should not return arrays
 
+        // TODO enable in following PR
+        ///// <summary>
+        ///// When requested using <see cref="DownloadTransferValidationOptions"/>, this value contains the CRC for the download blob range.
+        ///// This value may only become populated once the network stream is fully consumed. If this instance is accessed through
+        ///// <see cref="BlobDownloadResult"/>, the network stream has already been consumed. Otherwise, consume the content stream before
+        ///// checking this value.
+        ///// </summary>
+        //public byte[] ContentCrc { get; internal set; }
+
         /// <summary>
         /// Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob.
         /// </summary>

sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs

@@ -4,6 +4,8 @@
 using System;
 using System.ComponentModel;
 using System.IO;
+using System.Threading.Tasks;
+using Azure.Core;
 using Azure.Storage.Shared;
 
 namespace Azure.Storage.Blobs.Models
@@ -49,6 +51,15 @@ public class BlobDownloadInfo : IDisposable, IDownloadedContent
         /// </summary>
         public BlobDownloadDetails Details { get; internal set; }
 
+        // TODO enable in following PR
+        ///// <summary>
+        ///// Indicates some contents of <see cref="Details"/> are mixed into the response stream.
+        ///// They will not be set until <see cref="Content"/> has been fully consumed. These details
+        ///// will be extracted from the content stream by the library before the calling code can
+        ///// encounter them.
+        ///// </summary>
+        //public bool ExpectTrailingDetails { get; internal set; }
+
         /// <summary>
         /// Constructor.
         /// </summary>

sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs

@@ -1370,7 +1370,6 @@ internal async Task<Response<PageInfo>> UploadPagesInternal(
                     HttpRange range;
                     if (validationOptions != null &&
                         validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 &&
-                        validationOptions.PrecalculatedChecksum.IsEmpty &&
                         ClientSideEncryption == null) // don't allow feature combination
                     {
                         // report progress in terms of caller bytes, not encoded bytes
@@ -1379,10 +1378,14 @@ internal async Task<Response<PageInfo>> UploadPagesInternal(
                         range = new HttpRange(offset, (content?.Length - content?.Position) ?? null);
                         structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage;
                         content = content?.WithNoDispose().WithProgress(progressHandler);
-                        content = new StructuredMessageEncodingStream(
-                            content,
-                            Constants.StructuredMessage.DefaultSegmentContentLength,
-                            StructuredMessage.Flags.StorageCrc64);
+                        content = validationOptions.PrecalculatedChecksum.IsEmpty
+                            ? new StructuredMessageEncodingStream(
+                                content,
+                                Constants.StructuredMessage.DefaultSegmentContentLength,
+                                StructuredMessage.Flags.StorageCrc64)
+                            : new StructuredMessagePrecalculatedCrcWrapperStream(
+                                content,
+                                validationOptions.PrecalculatedChecksum.Span);
                         contentLength = (content?.Length - content?.Position) ?? 0;
                     }
                     else

sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs

@@ -47,7 +47,8 @@ internal class PartitionedDownloader
         /// </summary>
         private readonly StorageChecksumAlgorithm _validationAlgorithm;
         private readonly int _checksumSize;
-        private bool UseMasterCrc => _validationAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64;
+        // TODO disabling master crc temporarily. segment CRCs still handled.
+        private bool UseMasterCrc => false; // _validationAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64;
         private StorageCrc64HashAlgorithm _masterCrcCalculator = null;
 
         /// <summary>
@@ -211,8 +212,20 @@ public async Task<Response> DownloadToInternal(
 
                 // If the first segment was the entire blob, we'll copy that to
                 // the output stream and finish now
-                long initialLength = initialResponse.Value.Details.ContentLength;
-                long totalLength = ParseRangeTotalLength(initialResponse.Value.Details.ContentRange);
+                long initialLength;
+                long totalLength;
+                // Get blob content length downloaded from content range when available to handle transit encoding
+                if (string.IsNullOrWhiteSpace(initialResponse.Value.Details.ContentRange))
+                {
+                    initialLength = initialResponse.Value.Details.ContentLength;
+                    totalLength = 0;
+                }
+                else
+                {
+                    ContentRange recievedRange = ContentRange.Parse(initialResponse.Value.Details.ContentRange);
+                    initialLength = recievedRange.End.Value - recievedRange.Start.Value + 1;
+                    totalLength = recievedRange.Size.Value;
+                }
                 if (initialLength == totalLength)
                 {
                     await HandleOneShotDownload(initialResponse, destination, async, cancellationToken)
@@ -394,20 +407,6 @@ private async Task FinalizeDownloadInternal(
             }
         }
 
-        private static long ParseRangeTotalLength(string range)
-        {
-            if (range == null)
-            {
-                return 0;
-            }
-            int lengthSeparator = range.IndexOf("/", StringComparison.InvariantCultureIgnoreCase);
-            if (lengthSeparator == -1)
-            {
-                throw BlobErrors.ParsingFullHttpRangeFailed(range);
-            }
-            return long.Parse(range.Substring(lengthSeparator + 1), CultureInfo.InvariantCulture);
-        }
-
         private async Task CopyToInternal(
             Response<BlobDownloadStreamingResult> response,
             Stream destination,
@@ -416,7 +415,10 @@ private async Task CopyToInternal(
             CancellationToken cancellationToken)
         {
             CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
-            using IHasher hasher = ContentHasher.GetHasherFromAlgorithmId(_validationAlgorithm);
+            // if structured message, this crc is validated in the decoding process. don't decode it here.
+            using IHasher hasher = response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)
+                ? null
+                : ContentHasher.GetHasherFromAlgorithmId(_validationAlgorithm);
             using Stream rawSource = response.Value.Content;
             using Stream source = hasher != null
                 ? ChecksumCalculatingStream.GetReadStream(rawSource, hasher.AppendHash)
@@ -431,13 +433,13 @@ await source.CopyToInternal(
             if (hasher != null)
             {
                 hasher.GetFinalHash(checksumBuffer.Span);
-                (ReadOnlyMemory<byte> checksum, StorageChecksumAlgorithm _)
-                    = ContentHasher.GetResponseChecksumOrDefault(response.GetRawResponse());
-                if (!checksumBuffer.Span.SequenceEqual(checksum.Span))
-                {
-                    throw Errors.HashMismatchOnStreamedDownload(response.Value.Details.ContentRange);
+                    (ReadOnlyMemory<byte> checksum, StorageChecksumAlgorithm _)
+                        = ContentHasher.GetResponseChecksumOrDefault(response.GetRawResponse());
+                    if (!checksumBuffer.Span.SequenceEqual(checksum.Span))
+                    {
+                        throw Errors.HashMismatchOnStreamedDownload(response.Value.Details.ContentRange);
+                    }
                 }
-            }
         }
 
         private IEnumerable<HttpRange> GetRanges(long initialLength, long totalLength)

sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs

@@ -95,59 +95,6 @@ public override void TestAutoResolve()
         }
 
         #region Added Tests
-        [TestCaseSource("GetValidationAlgorithms")]
-        public async Task ExpectedDownloadStreamingStreamTypeReturned(StorageChecksumAlgorithm algorithm)
-        {
-            await using var test = await GetDisposingContainerAsync();
-
-            // Arrange
-            var data = GetRandomBuffer(Constants.KB);
-            BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName()));
-            using (var stream = new MemoryStream(data))
-            {
-                await blob.UploadAsync(stream);
-            }
-            // don't make options instance at all for no hash request
-            DownloadTransferValidationOptions transferValidation = algorithm == StorageChecksumAlgorithm.None
-                ? default
-                : new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm };
-
-            // Act
-            Response<BlobDownloadStreamingResult> response = await blob.DownloadStreamingAsync(new BlobDownloadOptions
-            {
-                TransferValidation = transferValidation,
-                Range = new HttpRange(length: data.Length)
-            });
-
-            // Assert
-            // validated stream is buffered
-            Assert.AreEqual(typeof(MemoryStream), response.Value.Content.GetType());
-        }
-
-        [Test]
-        public async Task ExpectedDownloadStreamingStreamTypeReturned_None()
-        {
-            await using var test = await GetDisposingContainerAsync();
-
-            // Arrange
-            var data = GetRandomBuffer(Constants.KB);
-            BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName()));
-            using (var stream = new MemoryStream(data))
-            {
-                await blob.UploadAsync(stream);
-            }
-
-            // Act
-            Response<BlobDownloadStreamingResult> response = await blob.DownloadStreamingAsync(new BlobDownloadOptions
-            {
-                Range = new HttpRange(length: data.Length)
-            });
-
-            // Assert
-            // unvalidated stream type is private; just check we didn't get back a buffered stream
-            Assert.AreNotEqual(typeof(MemoryStream), response.Value.Content.GetType());
-        }
-
         [Test]
         public virtual async Task OlderServiceVersionThrowsOnStructuredMessage()
         {

sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs

@@ -305,7 +305,7 @@ public Response<BlobDownloadStreamingResult> GetStream(HttpRange range, BlobRequ
                         ContentHash = new byte[] { 1, 2, 3 },
                         LastModified = DateTimeOffset.Now,
                         Metadata = new Dictionary<string, string>() { { "meta", "data" } },
-                        ContentRange = $"bytes {range.Offset}-{range.Offset + contentLength}/{_length}",
+                        ContentRange = $"bytes {range.Offset}-{range.Offset + contentLength - 1}/{_length}",
                         ETag = s_etag,
                         ContentEncoding = "test",
                         CacheControl = "test",

sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs

@@ -249,49 +249,17 @@ private async Task<int> DownloadInternal(bool async, CancellationToken cancellat
             response = await _downloadInternalFunc(range, _validationOptions, async, cancellationToken).ConfigureAwait(false);
 
             using Stream networkStream = response.Value.Content;
-
-            // The number of bytes we just downloaded.
-            long downloadSize = GetResponseRange(response.GetRawResponse()).Length.Value;
-
-            // The number of bytes we copied in the last loop.
-            int copiedBytes;
-
-            // Bytes we have copied so far.
-            int totalCopiedBytes = 0;
-
-            // Bytes remaining to copy.  It is save to truncate the long because we asked for a max of int _buffer size bytes.
-            int remainingBytes = (int)downloadSize;
-
-            do
-            {
-                if (async)
-                {
-                    copiedBytes = await networkStream.ReadAsync(
-                        buffer: _buffer,
-                        offset: totalCopiedBytes,
-                        count: remainingBytes,
-                        cancellationToken: cancellationToken).ConfigureAwait(false);
-                }
-                else
-                {
-                    copiedBytes = networkStream.Read(
-                        buffer: _buffer,
-                        offset: totalCopiedBytes,
-                        count: remainingBytes);
-                }
-
-                totalCopiedBytes += copiedBytes;
-                remainingBytes -= copiedBytes;
-            }
-            while (copiedBytes != 0);
+            // use stream copy to ensure consumption of any trailing metadata (e.g. structured message)
+            // allow buffer limits to catch the error of data size mismatch
+            int totalCopiedBytes = (int) await networkStream.CopyToInternal(new MemoryStream(_buffer), async, cancellationToken).ConfigureAwait((false));
 
             _bufferPosition = 0;
             _bufferLength = totalCopiedBytes;
             _length = GetBlobLengthFromResponse(response.GetRawResponse());
 
             // if we deferred transactional hash validation on download, validate now
             // currently we always defer but that may change
-            if (_validationOptions != default && _validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && !_validationOptions.AutoValidateChecksum)
+            if (_validationOptions != default && _validationOptions.ChecksumAlgorithm == StorageChecksumAlgorithm.MD5 && !_validationOptions.AutoValidateChecksum) // TODO better condition
             {
                 ContentHasher.AssertResponseHashMatch(_buffer, _bufferPosition, _bufferLength, _validationOptions.ChecksumAlgorithm, response.GetRawResponse());
             }