From bb1a15382eba5baee6731528404f47048abd5964 Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Wed, 20 Sep 2017 14:32:09 -0400 Subject: [PATCH 01/10] request b2_download_file_by_name if listing a single file with no versions --- src/duplicacy_b2client.go | 55 ++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/src/duplicacy_b2client.go b/src/duplicacy_b2client.go index c797b7f..2b4d023 100644 --- a/src/duplicacy_b2client.go +++ b/src/duplicacy_b2client.go @@ -82,7 +82,7 @@ func (client *B2Client) retry(backoff int, response *http.Response) int { return backoff } -func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int64, error) { +func (client *B2Client) call(url string, requestHeaders map[string]string, input interface{}) (io.ReadCloser, Header, int64, error) { var response *http.Response @@ -95,7 +95,7 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6 default: jsonInput, err := json.Marshal(input) if err != nil { - return nil, 0, err + return nil, nil, 0, err } inputReader = bytes.NewReader(jsonInput) case []byte: @@ -103,11 +103,14 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6 case int: method = "GET" inputReader = bytes.NewReader([]byte("")) + case nil: + method = "HEAD" + inputReader = bytes.NewReader([]byte("")) } request, err := http.NewRequest(method, url, inputReader) if err != nil { - return nil, 0, err + return nil, nil, 0, err } if url == B2AuthorizationURL { @@ -116,6 +119,10 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6 request.Header.Set("Authorization", client.AuthorizationToken) } + for k, v := range requestHeaders { + request.Header.Set(k, v) + } + if client.TestMode { r := rand.Float32() if r < 0.5 { @@ -132,11 +139,11 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6 backoff = client.retry(backoff, response) continue } - return nil, 0, err + return nil, nil, 0, err } if response.StatusCode < 300 { - return response.Body, response.ContentLength, nil + return response.Body, response.Header, response.ContentLength, nil } LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode) @@ -145,13 +152,13 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6 response.Body.Close() if response.StatusCode == 401 { if url == B2AuthorizationURL { - return nil, 0, fmt.Errorf("Authorization failure") + return nil, nil, 0, fmt.Errorf("Authorization failure") } client.AuthorizeAccount() continue } else if response.StatusCode == 403 { if !client.TestMode { - return nil, 0, fmt.Errorf("B2 cap exceeded") + return nil, nil, 0, fmt.Errorf("B2 cap exceeded") } continue } else if response.StatusCode == 429 || response.StatusCode == 408 { @@ -172,13 +179,13 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6 } if err := json.NewDecoder(response.Body).Decode(e); err != nil { - return nil, 0, err + return nil, nil, 0, err } - return nil, 0, e + return nil, nil, 0, e } - return nil, 0, fmt.Errorf("Maximum backoff reached") + return nil, nil, 0, fmt.Errorf("Maximum backoff reached") } type B2AuthorizeAccountOutput struct { @@ -287,12 +294,19 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc input["startFileName"] = startFileName input["maxFileCount"] = maxFileCount + url := client.APIURL + "/b2api/v1/b2_list_file_names" + requestHeaders := map[string]string{} + if includeVersions { + url = client.APIURL + "/b2api/v1/b2_list_file_versions" + } else if singleFile { + // handle a single file with no versions as a special case to download the first byte of the file + url = client.APIURL + "/b2api/v1/b2_download_file_by_name" + requestHeaders["Range"] = "bytes=0-0" + // HEAD request + input = nil + } for { - url := client.APIURL + "/b2api/v1/b2_list_file_names" - if includeVersions { - url = client.APIURL + "/b2api/v1/b2_list_file_versions" - } - readCloser, _, err := client.call(url, input) + readCloser, responseHeader, _, err := client.call(url, requestHeaders, input) if err != nil { return nil, err } @@ -302,6 +316,17 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc output := B2ListFileNamesOutput { } + if singleFile && !includeVersions { + // construct the B2Entry from the response headers of the download request + file := B2Entry + file.FileID = responseHeader().Get("X-Bz-File-Id") + file.FileName = responseHeader().Get("X-Bz-File-Name") + file.Action = responseHeader().Get("X-Bz-Action") + file.Size = responseHeader().Get("Content-Length") + file.UploadTimestamp = responseHeader().Get("X-Bz-Upload-Timestamp") + return []*B2Entry{file} + } + if err = json.NewDecoder(readCloser).Decode(&output); err != nil { return nil, err } From 3c03b566aebe9413fd8800799ea558ed03b2e047 Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Wed, 20 Sep 2017 14:32:09 -0400 Subject: [PATCH 02/10] request b2_download_file_by_name if listing a single file with no versions --- src/duplicacy_b2client.go | 55 ++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/src/duplicacy_b2client.go b/src/duplicacy_b2client.go index c797b7f..2b4d023 100644 --- a/src/duplicacy_b2client.go +++ b/src/duplicacy_b2client.go @@ -82,7 +82,7 @@ func (client *B2Client) retry(backoff int, response *http.Response) int { return backoff } -func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int64, error) { +func (client *B2Client) call(url string, requestHeaders map[string]string, input interface{}) (io.ReadCloser, Header, int64, error) { var response *http.Response @@ -95,7 +95,7 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6 default: jsonInput, err := json.Marshal(input) if err != nil { - return nil, 0, err + return nil, nil, 0, err } inputReader = bytes.NewReader(jsonInput) case []byte: @@ -103,11 +103,14 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6 case int: method = "GET" inputReader = bytes.NewReader([]byte("")) + case nil: + method = "HEAD" + inputReader = bytes.NewReader([]byte("")) } request, err := http.NewRequest(method, url, inputReader) if err != nil { - return nil, 0, err + return nil, nil, 0, err } if url == B2AuthorizationURL { @@ -116,6 +119,10 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6 request.Header.Set("Authorization", client.AuthorizationToken) } + for k, v := range requestHeaders { + request.Header.Set(k, v) + } + if client.TestMode { r := rand.Float32() if r < 0.5 { @@ -132,11 +139,11 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6 backoff = client.retry(backoff, response) continue } - return nil, 0, err + return nil, nil, 0, err } if response.StatusCode < 300 { - return response.Body, response.ContentLength, nil + return response.Body, response.Header, response.ContentLength, nil } LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode) @@ -145,13 +152,13 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6 response.Body.Close() if response.StatusCode == 401 { if url == B2AuthorizationURL { - return nil, 0, fmt.Errorf("Authorization failure") + return nil, nil, 0, fmt.Errorf("Authorization failure") } client.AuthorizeAccount() continue } else if response.StatusCode == 403 { if !client.TestMode { - return nil, 0, fmt.Errorf("B2 cap exceeded") + return nil, nil, 0, fmt.Errorf("B2 cap exceeded") } continue } else if response.StatusCode == 429 || response.StatusCode == 408 { @@ -172,13 +179,13 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6 } if err := json.NewDecoder(response.Body).Decode(e); err != nil { - return nil, 0, err + return nil, nil, 0, err } - return nil, 0, e + return nil, nil, 0, e } - return nil, 0, fmt.Errorf("Maximum backoff reached") + return nil, nil, 0, fmt.Errorf("Maximum backoff reached") } type B2AuthorizeAccountOutput struct { @@ -287,12 +294,19 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc input["startFileName"] = startFileName input["maxFileCount"] = maxFileCount + url := client.APIURL + "/b2api/v1/b2_list_file_names" + requestHeaders := map[string]string{} + if includeVersions { + url = client.APIURL + "/b2api/v1/b2_list_file_versions" + } else if singleFile { + // handle a single file with no versions as a special case to download the first byte of the file + url = client.APIURL + "/b2api/v1/b2_download_file_by_name" + requestHeaders["Range"] = "bytes=0-0" + // HEAD request + input = nil + } for { - url := client.APIURL + "/b2api/v1/b2_list_file_names" - if includeVersions { - url = client.APIURL + "/b2api/v1/b2_list_file_versions" - } - readCloser, _, err := client.call(url, input) + readCloser, responseHeader, _, err := client.call(url, requestHeaders, input) if err != nil { return nil, err } @@ -302,6 +316,17 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc output := B2ListFileNamesOutput { } + if singleFile && !includeVersions { + // construct the B2Entry from the response headers of the download request + file := B2Entry + file.FileID = responseHeader().Get("X-Bz-File-Id") + file.FileName = responseHeader().Get("X-Bz-File-Name") + file.Action = responseHeader().Get("X-Bz-Action") + file.Size = responseHeader().Get("Content-Length") + file.UploadTimestamp = responseHeader().Get("X-Bz-Upload-Timestamp") + return []*B2Entry{file} + } + if err = json.NewDecoder(readCloser).Decode(&output); err != nil { return nil, err } From b3d1eb36bde84a8c1a8c2fb51363d6bc6e8f2eaf Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Wed, 20 Sep 2017 22:38:35 -0400 Subject: [PATCH 03/10] fixes and filling out calls to b2_client.call --- src/duplicacy_b2client.go | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/duplicacy_b2client.go b/src/duplicacy_b2client.go index 2b4d023..c1866e6 100644 --- a/src/duplicacy_b2client.go +++ b/src/duplicacy_b2client.go @@ -82,7 +82,7 @@ func (client *B2Client) retry(backoff int, response *http.Response) int { return backoff } -func (client *B2Client) call(url string, requestHeaders map[string]string, input interface{}) (io.ReadCloser, Header, int64, error) { +func (client *B2Client) call(url string, requestHeaders map[string]string, input interface{}) (io.ReadCloser, http.Header, int64, error) { var response *http.Response @@ -119,8 +119,10 @@ func (client *B2Client) call(url string, requestHeaders map[string]string, input request.Header.Set("Authorization", client.AuthorizationToken) } - for k, v := range requestHeaders { - request.Header.Set(k, v) + if requestHeaders != nil { + for key, value := range requestHeaders { + request.Header.Set(key, value) + } } if client.TestMode { @@ -197,7 +199,7 @@ type B2AuthorizeAccountOutput struct { func (client *B2Client) AuthorizeAccount() (err error) { - readCloser, _, err := client.call(B2AuthorizationURL, make(map[string]string)) + readCloser, _, _, err := client.call(B2AuthorizationURL, make(map[string]string), make(map[string]string)) if err != nil { return err } @@ -231,7 +233,7 @@ func (client *B2Client) FindBucket(bucketName string) (err error) { url := client.APIURL + "/b2api/v1/b2_list_buckets" - readCloser, _, err := client.call(url, input) + readCloser, _, _, err := client.call(url, make(map[string]string), input) if err != nil { return err } @@ -318,13 +320,13 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc if singleFile && !includeVersions { // construct the B2Entry from the response headers of the download request - file := B2Entry - file.FileID = responseHeader().Get("X-Bz-File-Id") - file.FileName = responseHeader().Get("X-Bz-File-Name") - file.Action = responseHeader().Get("X-Bz-Action") - file.Size = responseHeader().Get("Content-Length") - file.UploadTimestamp = responseHeader().Get("X-Bz-Upload-Timestamp") - return []*B2Entry{file} + fileID := responseHeader.Get("X-Bz-File-Id") + fileName := responseHeader.Get("X-Bz-File-Name") + fileAction := responseHeader.Get("X-Bz-Action") + fileSize, _ := strconv.ParseInt(responseHeader.Get("Content-Length"), 0, 64) + fileUploadTimestamp, _ := strconv.ParseInt(responseHeader.Get("X-Bz-Upload-Timestamp"), 0, 64) + + return []*B2Entry {&B2Entry { fileID, fileName, fileAction, fileSize, fileUploadTimestamp }}, nil } if err = json.NewDecoder(readCloser).Decode(&output); err != nil { @@ -380,7 +382,7 @@ func (client *B2Client) DeleteFile(fileName string, fileID string) (err error) { input["fileId"] = fileID url := client.APIURL + "/b2api/v1/b2_delete_file_version" - readCloser, _, err := client.call(url, input) + readCloser, _, _, err := client.call(url, make(map[string]string), input) if err != nil { return err } @@ -400,7 +402,7 @@ func (client *B2Client) HideFile(fileName string) (fileID string, err error) { input["fileName"] = fileName url := client.APIURL + "/b2api/v1/b2_hide_file" - readCloser, _, err := client.call(url, input) + readCloser, _, _, err := client.call(url, make(map[string]string), input) if err != nil { return "", err } @@ -420,8 +422,8 @@ func (client *B2Client) HideFile(fileName string) (fileID string, err error) { func (client *B2Client) DownloadFile(filePath string) (io.ReadCloser, int64, error) { url := client.DownloadURL + "/file/" + client.BucketName + "/" + filePath - - return client.call(url, 0) + readCloser, _, len, err := client.call(url, make(map[string]string), 0) + return readCloser, len, err } type B2GetUploadArgumentOutput struct { @@ -435,7 +437,7 @@ func (client *B2Client) getUploadURL() (error) { input["bucketId"] = client.BucketID url := client.APIURL + "/b2api/v1/b2_get_upload_url" - readCloser, _, err := client.call(url, input) + readCloser, _, _, err := client.call(url, make(map[string]string), input) if err != nil { return err } From 2c2884abfb518e3c2ad0684a323ba16e9a6667cc Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Thu, 21 Sep 2017 03:18:08 -0400 Subject: [PATCH 04/10] fixes to fetching file info via b2_download_file_by_name --- src/duplicacy_b2client.go | 45 +++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/duplicacy_b2client.go b/src/duplicacy_b2client.go index c1866e6..6d01791 100644 --- a/src/duplicacy_b2client.go +++ b/src/duplicacy_b2client.go @@ -103,7 +103,7 @@ func (client *B2Client) call(url string, requestHeaders map[string]string, input case int: method = "GET" inputReader = bytes.NewReader([]byte("")) - case nil: + case bool: method = "HEAD" inputReader = bytes.NewReader([]byte("")) } @@ -295,20 +295,28 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc input["bucketId"] = client.BucketID input["startFileName"] = startFileName input["maxFileCount"] = maxFileCount + inputMapBool := true - url := client.APIURL + "/b2api/v1/b2_list_file_names" - requestHeaders := map[string]string{} - if includeVersions { - url = client.APIURL + "/b2api/v1/b2_list_file_versions" - } else if singleFile { - // handle a single file with no versions as a special case to download the first byte of the file - url = client.APIURL + "/b2api/v1/b2_download_file_by_name" - requestHeaders["Range"] = "bytes=0-0" - // HEAD request - input = nil - } for { - readCloser, responseHeader, _, err := client.call(url, requestHeaders, input) + url := client.APIURL + "/b2api/v1/b2_list_file_names" + requestHeaders := map[string]string{} + if includeVersions { + url = client.APIURL + "/b2api/v1/b2_list_file_versions" + } else if singleFile { + // handle a single file with no versions as a special case to download the first byte of the file + url = client.DownloadURL + "/file/" + client.BucketName + "/" + startFileName + requestHeaders["Range"] = "bytes=0-0" + // HEAD request + inputMapBool = false + } + var readCloser io.ReadCloser + var responseHeader http.Header + var err error + if inputMapBool { + readCloser, responseHeader, _, err = client.call(url, requestHeaders, input) + } else { + readCloser, responseHeader, _, err = client.call(url, requestHeaders, false) + } if err != nil { return nil, err } @@ -318,12 +326,13 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc output := B2ListFileNamesOutput { } - if singleFile && !includeVersions { + if !inputMapBool && singleFile && !includeVersions { // construct the B2Entry from the response headers of the download request - fileID := responseHeader.Get("X-Bz-File-Id") - fileName := responseHeader.Get("X-Bz-File-Name") - fileAction := responseHeader.Get("X-Bz-Action") - fileSize, _ := strconv.ParseInt(responseHeader.Get("Content-Length"), 0, 64) + fileID := responseHeader.Get("x-bz-file-id") + fileName := responseHeader.Get("x-bz-file-name") + fileAction := "upload" + rangeString := responseHeader.Get("Content-Range") + fileSize, _ := strconv.ParseInt(rangeString[strings.Index(rangeString, "/")+1:], 0, 64) fileUploadTimestamp, _ := strconv.ParseInt(responseHeader.Get("X-Bz-Upload-Timestamp"), 0, 64) return []*B2Entry {&B2Entry { fileID, fileName, fileAction, fileSize, fileUploadTimestamp }}, nil From be2c3931cd6196f400c8ea20273f4e885ba147a1 Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Sun, 1 Oct 2017 21:16:45 -0400 Subject: [PATCH 05/10] pass in the http request method rather than switching on input type --- src/duplicacy_b2client.go | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/duplicacy_b2client.go b/src/duplicacy_b2client.go index 083e14a..cc47b5a 100644 --- a/src/duplicacy_b2client.go +++ b/src/duplicacy_b2client.go @@ -81,14 +81,24 @@ func (client *B2Client) retry(backoff int, response *http.Response) int { return backoff } -func (client *B2Client) call(url string, requestHeaders map[string]string, input interface{}) (io.ReadCloser, http.Header, int64, error) { +func (client *B2Client) call(url string, method string, requestHeaders map[string]string, input interface{}) (io.ReadCloser, http.Header, int64, error) { + + switch method { + case http.MethodGet: + break + case http.MethodHead: + break + case http.MethodPost: + break + default: + return nil, nil, 0, fmt.Errorf("unhandled http request method: "+method) + } var response *http.Response backoff := 0 for i := 0; i < 8; i++ { var inputReader *bytes.Reader - method := "POST" switch input.(type) { default: @@ -100,10 +110,6 @@ func (client *B2Client) call(url string, requestHeaders map[string]string, input case []byte: inputReader = bytes.NewReader(input.([]byte)) case int: - method = "GET" - inputReader = bytes.NewReader([]byte("")) - case bool: - method = "HEAD" inputReader = bytes.NewReader([]byte("")) } @@ -197,7 +203,7 @@ type B2AuthorizeAccountOutput struct { func (client *B2Client) AuthorizeAccount() (err error) { - readCloser, _, _, err := client.call(B2AuthorizationURL, make(map[string]string), make(map[string]string)) + readCloser, _, _, err := client.call(B2AuthorizationURL, http.MethodPost, make(map[string]string), make(map[string]string)) if err != nil { return err } @@ -231,7 +237,7 @@ func (client *B2Client) FindBucket(bucketName string) (err error) { url := client.APIURL + "/b2api/v1/b2_list_buckets" - readCloser, _, _, err := client.call(url, make(map[string]string), input) + readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input) if err != nil { return err } @@ -311,9 +317,9 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc var responseHeader http.Header var err error if inputMapBool { - readCloser, responseHeader, _, err = client.call(url, requestHeaders, input) + readCloser, responseHeader, _, err = client.call(url, http.MethodPost, requestHeaders, input) } else { - readCloser, responseHeader, _, err = client.call(url, requestHeaders, false) + readCloser, responseHeader, _, err = client.call(url, http.MethodHead, requestHeaders, 0) } if err != nil { return nil, err @@ -388,7 +394,7 @@ func (client *B2Client) DeleteFile(fileName string, fileID string) (err error) { input["fileId"] = fileID url := client.APIURL + "/b2api/v1/b2_delete_file_version" - readCloser, _, _, err := client.call(url, make(map[string]string), input) + readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input) if err != nil { return err } @@ -408,7 +414,7 @@ func (client *B2Client) HideFile(fileName string) (fileID string, err error) { input["fileName"] = fileName url := client.APIURL + "/b2api/v1/b2_hide_file" - readCloser, _, _, err := client.call(url, make(map[string]string), input) + readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input) if err != nil { return "", err } @@ -429,7 +435,7 @@ func (client *B2Client) DownloadFile(filePath string) (io.ReadCloser, int64, err url := client.DownloadURL + "/file/" + client.BucketName + "/" + filePath - readCloser, _, len, err := client.call(url, make(map[string]string), 0) + readCloser, _, len, err := client.call(url, http.MethodGet, make(map[string]string), 0) return readCloser, len, err } @@ -444,7 +450,7 @@ func (client *B2Client) getUploadURL() error { input["bucketId"] = client.BucketID url := client.APIURL + "/b2api/v1/b2_get_upload_url" - readCloser, _, _, err := client.call(url, make(map[string]string), input) + readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input) if err != nil { return err } From f57fe55543b4cecce20a131600b25d11eaac3083 Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Sun, 1 Oct 2017 23:07:25 -0400 Subject: [PATCH 06/10] cleanup, simplified ListFileNames invoke of "call" --- src/duplicacy_b2client.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/duplicacy_b2client.go b/src/duplicacy_b2client.go index cc47b5a..934b095 100644 --- a/src/duplicacy_b2client.go +++ b/src/duplicacy_b2client.go @@ -203,7 +203,7 @@ type B2AuthorizeAccountOutput struct { func (client *B2Client) AuthorizeAccount() (err error) { - readCloser, _, _, err := client.call(B2AuthorizationURL, http.MethodPost, make(map[string]string), make(map[string]string)) + readCloser, _, _, err := client.call(B2AuthorizationURL, http.MethodPost, nil, make(map[string]string)) if err != nil { return err } @@ -237,7 +237,7 @@ func (client *B2Client) FindBucket(bucketName string) (err error) { url := client.APIURL + "/b2api/v1/b2_list_buckets" - readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input) + readCloser, _, _, err := client.call(url, http.MethodPost, nil, input) if err != nil { return err } @@ -299,11 +299,13 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc input["bucketId"] = client.BucketID input["startFileName"] = startFileName input["maxFileCount"] = maxFileCount - inputMapBool := true for { url := client.APIURL + "/b2api/v1/b2_list_file_names" requestHeaders := map[string]string{} + requestMethod := http.MethodPost + var requestInput interface{} + requestInput = input if includeVersions { url = client.APIURL + "/b2api/v1/b2_list_file_versions" } else if singleFile { @@ -311,16 +313,13 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc url = client.DownloadURL + "/file/" + client.BucketName + "/" + startFileName requestHeaders["Range"] = "bytes=0-0" // HEAD request - inputMapBool = false + requestMethod = http.MethodHead + requestInput = 0 } var readCloser io.ReadCloser var responseHeader http.Header var err error - if inputMapBool { - readCloser, responseHeader, _, err = client.call(url, http.MethodPost, requestHeaders, input) - } else { - readCloser, responseHeader, _, err = client.call(url, http.MethodHead, requestHeaders, 0) - } + readCloser, responseHeader, _, err = client.call(url, requestMethod, requestHeaders, requestInput) if err != nil { return nil, err } @@ -329,7 +328,7 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc output := B2ListFileNamesOutput{} - if !inputMapBool && singleFile && !includeVersions { + if singleFile && !includeVersions { // construct the B2Entry from the response headers of the download request fileID := responseHeader.Get("x-bz-file-id") fileName := responseHeader.Get("x-bz-file-name") From 078464499614114c25e5f3c98a6759f8da6a9c9f Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Sun, 1 Oct 2017 23:08:25 -0400 Subject: [PATCH 07/10] goimports --- src/duplicacy_b2client.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/duplicacy_b2client.go b/src/duplicacy_b2client.go index 934b095..6cf624e 100644 --- a/src/duplicacy_b2client.go +++ b/src/duplicacy_b2client.go @@ -84,14 +84,14 @@ func (client *B2Client) retry(backoff int, response *http.Response) int { func (client *B2Client) call(url string, method string, requestHeaders map[string]string, input interface{}) (io.ReadCloser, http.Header, int64, error) { switch method { - case http.MethodGet: - break - case http.MethodHead: - break - case http.MethodPost: - break - default: - return nil, nil, 0, fmt.Errorf("unhandled http request method: "+method) + case http.MethodGet: + break + case http.MethodHead: + break + case http.MethodPost: + break + default: + return nil, nil, 0, fmt.Errorf("unhandled http request method: " + method) } var response *http.Response From 04debec0a1062ad200d42f44094be58e48c9671e Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Mon, 2 Oct 2017 02:04:30 -0400 Subject: [PATCH 08/10] request last byte (handles empty files), handle 404 and 416 errors, response header error checking --- src/duplicacy_b2client.go | 46 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/src/duplicacy_b2client.go b/src/duplicacy_b2client.go index 6cf624e..68aac56 100644 --- a/src/duplicacy_b2client.go +++ b/src/duplicacy_b2client.go @@ -168,6 +168,15 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin return nil, nil, 0, fmt.Errorf("B2 cap exceeded") } continue + } else if response.StatusCode == 404 { + if http.MethodHead == method { + return nil, nil, 0, fmt.Errorf("URL request '%s' returned status code %d", url, response.StatusCode) + } + } else if response.StatusCode == 416 { + if http.MethodHead == method { + // 416 Requested Range Not Satisfiable + return nil, nil, 0, fmt.Errorf("URL request '%s' returned status code %d", url, response.StatusCode) + } } else if response.StatusCode == 429 || response.StatusCode == 408 { backoff = client.retry(backoff, response) continue @@ -309,9 +318,10 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc if includeVersions { url = client.APIURL + "/b2api/v1/b2_list_file_versions" } else if singleFile { - // handle a single file with no versions as a special case to download the first byte of the file + // handle a single file with no versions as a special case to download the last byte of the file url = client.DownloadURL + "/file/" + client.BucketName + "/" + startFileName - requestHeaders["Range"] = "bytes=0-0" + // requesting byte -1 works for empty files where 0-0 fails with a 416 error + requestHeaders["Range"] = "bytes=-1" // HEAD request requestMethod = http.MethodHead requestInput = 0 @@ -329,12 +339,42 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc output := B2ListFileNamesOutput{} if singleFile && !includeVersions { + if responseHeader == nil { + return nil, fmt.Errorf("b2_download_file_by_name did not return headers") + } + requiredHeaders := []string{ + "x-bz-file-id", + "x-bz-file-name", + } + missingKeys := []string{} + for _,headerKey := range requiredHeaders { + if "" == responseHeader.Get(headerKey) { + missingKeys = append(missingKeys, headerKey) + } + } + if len(missingKeys) > 0 { + return nil, fmt.Errorf("b2_download_file_by_name missing headers: %s", missingKeys) + } // construct the B2Entry from the response headers of the download request fileID := responseHeader.Get("x-bz-file-id") fileName := responseHeader.Get("x-bz-file-name") fileAction := "upload" + // byte range that is returned: "bytes #-#/# rangeString := responseHeader.Get("Content-Range") - fileSize, _ := strconv.ParseInt(rangeString[strings.Index(rangeString, "/")+1:], 0, 64) + // total file size; 1 if file has content, 0 if it's empty + lengthString := responseHeader.Get("Content-Length") + var fileSize int64 + if "" != rangeString { + fileSize, _ = strconv.ParseInt(rangeString[strings.Index(rangeString, "/")+1:], 0, 64) + } else if "" != lengthString { + // this should only execute if the requested file is empty and the range request didn't result in a Content-Range header + fileSize, _ = strconv.ParseInt(lengthString, 0, 64) + if fileSize != 0 { + return nil, fmt.Errorf("b2_download_file_by_name returned non-zero file length") + } + } else { + return nil, fmt.Errorf("could not parse b2_download_file_by_name headers") + } fileUploadTimestamp, _ := strconv.ParseInt(responseHeader.Get("X-Bz-Upload-Timestamp"), 0, 64) return []*B2Entry{&B2Entry{fileID, fileName, fileAction, fileSize, fileUploadTimestamp}}, nil From f044d37b2853c79b279b34c7571b09d4f5ce5bf9 Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Mon, 2 Oct 2017 02:08:34 -0400 Subject: [PATCH 09/10] goimports --- src/duplicacy_b2client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/duplicacy_b2client.go b/src/duplicacy_b2client.go index 68aac56..d721e08 100644 --- a/src/duplicacy_b2client.go +++ b/src/duplicacy_b2client.go @@ -343,11 +343,11 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc return nil, fmt.Errorf("b2_download_file_by_name did not return headers") } requiredHeaders := []string{ - "x-bz-file-id", + "x-bz-file-id", "x-bz-file-name", } missingKeys := []string{} - for _,headerKey := range requiredHeaders { + for _, headerKey := range requiredHeaders { if "" == responseHeader.Get(headerKey) { missingKeys = append(missingKeys, headerKey) } From be89d8d0dc4084b79c1173865b36fb7144e0a5b3 Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Tue, 3 Oct 2017 00:10:57 -0400 Subject: [PATCH 10/10] treat 404 as file missing instead of generic error --- src/duplicacy_b2client.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/duplicacy_b2client.go b/src/duplicacy_b2client.go index d721e08..e889e5e 100644 --- a/src/duplicacy_b2client.go +++ b/src/duplicacy_b2client.go @@ -170,7 +170,8 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin continue } else if response.StatusCode == 404 { if http.MethodHead == method { - return nil, nil, 0, fmt.Errorf("URL request '%s' returned status code %d", url, response.StatusCode) + LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode) + return nil, nil, 0, nil } } else if response.StatusCode == 416 { if http.MethodHead == method { @@ -340,7 +341,8 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc if singleFile && !includeVersions { if responseHeader == nil { - return nil, fmt.Errorf("b2_download_file_by_name did not return headers") + LOG_DEBUG("BACKBLAZE_LIST", "b2_download_file_by_name did not return headers") + return []*B2Entry{}, nil } requiredHeaders := []string{ "x-bz-file-id",