Fix hanging blob creation (#351)

* fix hanging blob creation * remove unnecessary try-catch
microsoft · Mar 15, 2024 · 36e2ff0 · 36e2ff0
1 parent 7921c1e
commit 36e2ff0
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 4 deletions.
diff --git a/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/AzureStorageDevice.cs b/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/AzureStorageDevice.cs
@@ -429,7 +429,7 @@ public override void WriteAsync(IntPtr sourceAddress, int segmentId, ulong desti
 
                     // If no blob exists for the segment, we must first create the segment asynchronouly. (Create call takes ~70 ms by measurement)
                     // After creation is done, we can call write.
-                    _ = entry.CreateAsync(size, pageBlob);
+                    _ = entry.CreateAsync(size, pageBlob, id);
                 }
                 // Otherwise, some other thread beat us to it. Okay to use their blobs.
                 blobEntry = this.blobs[segmentId];

diff --git a/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/BlobEntry.cs b/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/BlobEntry.cs
@@ -55,7 +55,7 @@ public BlobEntry(AzureStorageDevice azureStorageDevice)
         /// </summary>
         /// <param name="size">maximum size of the blob</param>
         /// <param name="pageBlob">The page blob to create</param>
-        public async Task CreateAsync(long size, BlobUtilsV12.PageBlobClients pageBlob)
+        public async Task CreateAsync(long size, BlobUtilsV12.PageBlobClients pageBlob, long id)
         {
             if (this.waitingCount != 0)
             {
@@ -67,12 +67,17 @@ await this.azureStorageDevice.BlobManager.PerformWithRetriesAsync(
                 true,
                 "PageBlobClient.CreateAsync",
                 "CreateDevice",
-                "",
+                $"id={id}",
                 pageBlob.Default.Name,
                 3000,
                 true,
                 async (numAttempts) =>
                 {
+                    if (this.ETag != default)
+                    {
+                        return 1; // blob was already created by previous attempt
+                    }
+
                     var client = (numAttempts > 1) ? pageBlob.Default : pageBlob.Aggressive;
 
                     var response = await client.CreateAsync(

diff --git a/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/StorageOperations.cs b/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/StorageOperations.cs
@@ -101,7 +101,7 @@ public async Task PerformWithRetriesAsync(
                         }
                         continue;
                     }
-                    catch (Azure.RequestFailedException ex) when (BlobUtilsV12.PreconditionFailed(ex) && readETagAsync != null)
+                    catch (Azure.RequestFailedException ex) when (BlobUtilsV12.PreconditionFailed(ex) && readETagAsync != null && numAttempts < BlobManager.MaxRetries)
                     {
                         this.StorageTracer?.FasterStorageProgress($"storage operation {name} ({intent}) failed precondition on attempt {numAttempts}; target={target} latencyMs={stopwatch.Elapsed.TotalMilliseconds:F1} {details}");
                         mustReadETagFirst = true;