From 8e9caea201f13980614139710c2a02ad4c876a77 Mon Sep 17 00:00:00 2001 From: Gilbert Chen Date: Tue, 7 May 2019 22:35:51 -0400 Subject: [PATCH] Retry on broken pipe in Azure backend Azure sometimes disconnect the connection randomly when uploading files. The returned error was 'broken pipe' but this error is wrapped deep in multiple levels of errors so we have to check the error string instead. --- src/duplicacy_azurestorage.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/duplicacy_azurestorage.go b/src/duplicacy_azurestorage.go index fad919b..e5f2413 100644 --- a/src/duplicacy_azurestorage.go +++ b/src/duplicacy_azurestorage.go @@ -166,9 +166,21 @@ func (storage *AzureStorage) DownloadFile(threadIndex int, filePath string, chun // UploadFile writes 'content' to the file at 'filePath'. func (storage *AzureStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) { - reader := CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.containers)) - blob := storage.containers[threadIndex].GetBlobReference(filePath) - return blob.CreateBlockBlobFromReader(reader, nil) + + tries := 0 + + for { + reader := CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.containers)) + blob := storage.containers[threadIndex].GetBlobReference(filePath) + err = blob.CreateBlockBlobFromReader(reader, nil) + + if err == nil || !strings.Contains(err.Error(), "write: broken pipe") || tries >= 3 { + return err + } + + LOG_INFO("AZURE_RETRY", "Connection unexpectedly terminated: %v; retrying", err) + tries++ + } }