Bump version to 2.4.1

When copying to an RSA-encrypted storage, we relied on the RSA encryption
version to determine if a chunk is a snapshot chunk or a file chunk.  This is
wrong when the source storage is not encrypted or not RSA-encrypted.  There
is a more reliable to determine if a chunk is a snapshot chunk or not.

duplicacy/duplicacy_main.go

@@ -2059,7 +2059,7 @@ func main() {
 	app.Name = "duplicacy"
 	app.HelpName = "duplicacy"
 	app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
-	app.Version = "2.3.0" + " (" + GitCommit + ")"
+	app.Version = "2.4.1" + " (" + GitCommit + ")"
 
 	// If the program is interrupted, call the RunAtError function.
 	c := make(chan os.Signal, 1)

src/duplicacy_b2client.go

@@ -75,7 +75,7 @@ func B2Escape(path string) string {
 	return strings.Join(components, "/")
 }
 
-func NewB2Client(applicationKeyID string, applicationKey string, storageDir string, threads int) *B2Client {
+func NewB2Client(applicationKeyID string, applicationKey string, downloadURL string, storageDir string, threads int) *B2Client {
 
 	for storageDir != "" && storageDir[0] == '/' {
 		storageDir = storageDir[1:]
@@ -95,6 +95,7 @@ func NewB2Client(applicationKeyID string, applicationKey string, storageDir stri
 		HTTPClient:       http.DefaultClient,
 		ApplicationKeyID: applicationKeyID,
 		ApplicationKey:   applicationKey,
+		DownloadURL:      downloadURL,
 		StorageDir:       storageDir,
 		UploadURLs:       make([]string, threads),
 		UploadTokens:     make([]string, threads),
@@ -325,7 +326,10 @@ func (client *B2Client) AuthorizeAccount(threadIndex int) (err error, allowed bo
 
 	client.AuthorizationToken = output.AuthorizationToken
 	client.APIURL = output.APIURL
-	client.DownloadURL = output.DownloadURL
+	if client.DownloadURL == "" {
+		client.DownloadURL = output.DownloadURL
+	}
+	LOG_INFO("BACKBLAZE_URL", "download URL is: %s", client.DownloadURL)
 	client.IsAuthorized = true
 
 	client.LastAuthorizationTime = time.Now().Unix()
@@ -413,16 +417,16 @@ func (client *B2Client) ListFileNames(threadIndex int, startFileName string, sin
 	input["prefix"] = client.StorageDir
 
 	for {
-		url := client.getAPIURL() + "/b2api/v1/b2_list_file_names"
+		apiURL := client.getAPIURL() + "/b2api/v1/b2_list_file_names"
 		requestHeaders := map[string]string{}
 		requestMethod := http.MethodPost
 		var requestInput interface{}
 		requestInput = input
 		if includeVersions {
-			url = client.getAPIURL() + "/b2api/v1/b2_list_file_versions"
+			apiURL = client.getAPIURL() + "/b2api/v1/b2_list_file_versions"
 		} else if singleFile {
 			// handle a single file with no versions as a special case to download the last byte of the file
-			url = client.getDownloadURL() + "/file/" + client.BucketName + "/" + B2Escape(client.StorageDir + startFileName)
+			apiURL = client.getDownloadURL() + "/file/" + client.BucketName + "/" + B2Escape(client.StorageDir + startFileName)
 			// requesting byte -1 works for empty files where 0-0 fails with a 416 error
 			requestHeaders["Range"] = "bytes=-1"
 			// HEAD request
@@ -432,7 +436,7 @@ func (client *B2Client) ListFileNames(threadIndex int, startFileName string, sin
 		var readCloser io.ReadCloser
 		var responseHeader http.Header
 		var err error
-		readCloser, responseHeader, _, err = client.call(threadIndex, url, requestMethod, requestHeaders, requestInput)
+		readCloser, responseHeader, _, err = client.call(threadIndex, apiURL, requestMethod, requestHeaders, requestInput)
 		if err != nil {
 			return nil, err
 		}
@@ -445,7 +449,7 @@ func (client *B2Client) ListFileNames(threadIndex int, startFileName string, sin
 
 		if singleFile && !includeVersions {
 			if responseHeader == nil {
-				LOG_DEBUG("BACKBLAZE_LIST", "%s did not return headers", url)
+				LOG_DEBUG("BACKBLAZE_LIST", "%s did not return headers", apiURL)
 				return []*B2Entry{}, nil
 			}
 			requiredHeaders := []string{
@@ -459,11 +463,17 @@ func (client *B2Client) ListFileNames(threadIndex int, startFileName string, sin
 				}
 			}
 			if len(missingKeys) > 0 {
-				return nil, fmt.Errorf("%s missing headers: %s", url, missingKeys)
+				return nil, fmt.Errorf("%s missing headers: %s", apiURL, missingKeys)
 			}
 			// construct the B2Entry from the response headers of the download request
 			fileID := responseHeader.Get("x-bz-file-id")
 			fileName := responseHeader.Get("x-bz-file-name")
+			unescapedFileName, err := url.QueryUnescape(fileName)
+			if err == nil {
+				fileName = unescapedFileName
+			} else {
+				LOG_WARN("BACKBLAZE_UNESCAPE", "Failed to unescape the file name %s", fileName)
+			}
 			fileAction := "upload"
 			// byte range that is returned: "bytes #-#/#
 			rangeString := responseHeader.Get("Content-Range")
@@ -476,10 +486,10 @@ func (client *B2Client) ListFileNames(threadIndex int, startFileName string, sin
 				// this should only execute if the requested file is empty and the range request didn't result in a Content-Range header
 				fileSize, _ = strconv.ParseInt(lengthString, 0, 64)
 				if fileSize != 0 {
-					return nil, fmt.Errorf("%s returned non-zero file length", url)
+					return nil, fmt.Errorf("%s returned non-zero file length", apiURL)
 				}
 			} else {
-				return nil, fmt.Errorf("could not parse headers returned by %s", url)
+				return nil, fmt.Errorf("could not parse headers returned by %s", apiURL)
 			}
 			fileUploadTimestamp, _ := strconv.ParseInt(responseHeader.Get("X-Bz-Upload-Timestamp"), 0, 64)
 

src/duplicacy_b2client_test.go

@@ -37,7 +37,7 @@ func createB2ClientForTest(t *testing.T) (*B2Client, string) {
 		return nil, ""
 	}
 
-	return NewB2Client(b2["account"], b2["key"], b2["directory"], 1), b2["bucket"]
+	return NewB2Client(b2["account"], b2["key"], "", b2["directory"], 1), b2["bucket"]
 
 }
 

src/duplicacy_b2storage.go

@@ -15,9 +15,9 @@ type B2Storage struct {
 }
 
 // CreateB2Storage creates a B2 storage object.
-func CreateB2Storage(accountID string, applicationKey string, bucket string, storageDir string, threads int) (storage *B2Storage, err error) {
+func CreateB2Storage(accountID string, applicationKey string, downloadURL string, bucket string, storageDir string, threads int) (storage *B2Storage, err error) {
 
-	client := NewB2Client(accountID, applicationKey, storageDir, threads)
+	client := NewB2Client(accountID, applicationKey, downloadURL, storageDir, threads)
 
 	err, _ = client.AuthorizeAccount(0)
 	if err != nil {
@@ -204,7 +204,6 @@ func (storage *B2Storage) GetFileInfo(threadIndex int, filePath string) (exist b
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *B2Storage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 
-	filePath = strings.Replace(filePath, " ", "%20", -1)
 	readCloser, _, err := storage.client.DownloadFile(threadIndex, filePath)
 	if err != nil {
 		return err
@@ -218,7 +217,6 @@ func (storage *B2Storage) DownloadFile(threadIndex int, filePath string, chunk *
 
 // UploadFile writes 'content' to the file at 'filePath'.
 func (storage *B2Storage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
-	filePath = strings.Replace(filePath, " ", "%20", -1)
 	return storage.client.UploadFile(threadIndex, filePath, content, storage.UploadRateLimit/storage.client.Threads)
 }
 

src/duplicacy_backupmanager.go

@@ -1626,6 +1626,9 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 		return true
 	}
 
+	// These two maps store hashes of chunks in the source and destination storages, respectively.  Note that
+	// the value of 'chunks' is used to indicated if the chunk is a snapshot chunk, while the value of 'otherChunks' 
+	// is not used.
 	chunks := make(map[string]bool)
 	otherChunks := make(map[string]bool)
 
@@ -1638,21 +1641,15 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 		LOG_TRACE("SNAPSHOT_COPY", "Copying snapshot %s at revision %d", snapshot.ID, snapshot.Revision)
 
 		for _, chunkHash := range snapshot.FileSequence {
-			if _, found := chunks[chunkHash]; !found {
+			chunks[chunkHash] = true  // The chunk is a snapshot chunk
-				chunks[chunkHash] = true
-			}
 		}
 
 		for _, chunkHash := range snapshot.ChunkSequence {
-			if _, found := chunks[chunkHash]; !found {
+			chunks[chunkHash] = true  // The chunk is a snapshot chunk
-				chunks[chunkHash] = true
-			}
 		}
 
 		for _, chunkHash := range snapshot.LengthSequence {
-			if _, found := chunks[chunkHash]; !found {
+			chunks[chunkHash] = true  // The chunk is a snapshot chunk
-				chunks[chunkHash] = true
-			}
 		}
 
 		description := manager.SnapshotManager.DownloadSequence(snapshot.ChunkSequence)
@@ -1665,9 +1662,11 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 
 		for _, chunkHash := range snapshot.ChunkHashes {
 			if _, found := chunks[chunkHash]; !found {
-				chunks[chunkHash] = true
+				chunks[chunkHash] = false  // The chunk is a file chunk
 			}
 		}
+
+		snapshot.ChunkHashes = nil
 	}
 
 	otherChunkFiles, otherChunkSizes := otherManager.SnapshotManager.ListAllFiles(otherManager.storage, "chunks/")
@@ -1719,7 +1718,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 	totalSkipped := 0
 	chunkIndex := 0
 
-	for chunkHash := range chunks {
+	for chunkHash, isSnapshot := range chunks {
 		chunkIndex++
 		chunkID := manager.config.GetChunkIDFromHash(chunkHash)
 		newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash)
@@ -1730,7 +1729,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 			newChunk := otherManager.config.GetChunk()
 			newChunk.Reset(true)
 			newChunk.Write(chunk.GetBytes())
-			newChunk.encryptionVersion = chunk.encryptionVersion
+			newChunk.isSnapshot = isSnapshot
 			chunkUploader.StartChunk(newChunk, chunkIndex)
 			totalCopied++
 		} else {

src/duplicacy_chunk.go

@@ -63,12 +63,14 @@ type Chunk struct {
 	config *Config // Every chunk is associated with a Config object.  Which hashing algorithm to use is determined
 	// by the config
 
-	encryptionVersion byte // The version type in the encrytion header
+	isSnapshot bool // Indicates if the chunk is a snapshot chunk (instead of a file chunk).  This is only used by RSA
+	                // encryption, where a snapshot chunk is not encrypted by RSA
 }
 
 // Magic word to identify a duplicacy format encrypted file, plus a version number.
 var ENCRYPTION_HEADER = "duplicacy\000"
 
+// RSA encrypted chunks start with "duplicacy\002"
 var ENCRYPTION_VERSION_RSA byte = 2
 
 // CreateChunk creates a new chunk.
@@ -119,6 +121,7 @@ func (chunk *Chunk) Reset(hashNeeded bool) {
 	chunk.hash = nil
 	chunk.id = ""
 	chunk.size = 0
+	chunk.isSnapshot = false
 }
 
 // Write implements the Writer interface.
@@ -193,8 +196,8 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string, isSnapsh
 
 		key := encryptionKey
 		usingRSA := false
-		if chunk.config.rsaPublicKey != nil && (!isSnapshot || chunk.encryptionVersion == ENCRYPTION_VERSION_RSA) {
+		// Enable RSA encryption only when the chunk is not a snapshot chunk
-			// If the chunk is not a snpashot chunk, we attempt to encrypt it with the RSA publick key if there is one
+		if chunk.config.rsaPublicKey != nil && !isSnapshot && !chunk.isSnapshot {
 			randomKey := make([]byte, 32)
 			_, err := rand.Read(randomKey)
 			if err != nil {
@@ -321,8 +324,6 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
 	chunk.buffer, encryptedBuffer = encryptedBuffer, chunk.buffer
 	headerLength := len(ENCRYPTION_HEADER)
 
-	chunk.encryptionVersion = 0
-
 	if len(encryptionKey) > 0 {
 
 		key := encryptionKey
@@ -347,12 +348,12 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
 			return fmt.Errorf("The storage doesn't seem to be encrypted")
 		}
 
-		chunk.encryptionVersion = encryptedBuffer.Bytes()[headerLength-1]
+		encryptionVersion := encryptedBuffer.Bytes()[headerLength-1]
-		if chunk.encryptionVersion != 0 && chunk.encryptionVersion != ENCRYPTION_VERSION_RSA {
+		if encryptionVersion != 0 && encryptionVersion != ENCRYPTION_VERSION_RSA {
-			return fmt.Errorf("Unsupported encryption version %d", chunk.encryptionVersion)
+			return fmt.Errorf("Unsupported encryption version %d", encryptionVersion)
 		}
 
-		if chunk.encryptionVersion == ENCRYPTION_VERSION_RSA {
+		if encryptionVersion == ENCRYPTION_VERSION_RSA {
 			if chunk.config.rsaPrivateKey == nil {
 				LOG_ERROR("CHUNK_DECRYPT", "An RSA private key is required to decrypt the chunk")
 				return fmt.Errorf("An RSA private key is required to decrypt the chunk")

src/duplicacy_config.go

@@ -588,6 +588,11 @@ func (config *Config) loadRSAPublicKey(keyFile string) {
 // loadRSAPrivateKey loads the specifed private key file for decrypting file chunks
 func (config *Config) loadRSAPrivateKey(keyFile string, passphrase string) {
 
+	if config.rsaPublicKey == nil {
+		LOG_ERROR("RSA_PUBLIC", "The storage was not encrypted by an RSA key")
+		return
+	}
+
 	encodedKey, err := ioutil.ReadFile(keyFile)
 	if err != nil {
 		LOG_ERROR("RSA_PRIVATE", "Failed to read the private key file: %v", err)

src/duplicacy_sftpstorage.go

@@ -91,7 +91,7 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
 		storageDir:      storageDir,
 		minimumNesting:  minimumNesting,
 		numberOfThreads: threads,
-		numberOfTries:   6,
+		numberOfTries:   8,
 		serverAddress:   serverAddress,
 		sftpConfig:      sftpConfig,
 	}
@@ -129,22 +129,19 @@ func (storage *SFTPStorage) retry(f func () error) error {
 			delay *= 2
 
 			storage.clientLock.Lock()
-			if storage.client != nil {
-				storage.client.Close()
-				storage.client = nil
-			}
-
 			connection, err := ssh.Dial("tcp", storage.serverAddress, storage.sftpConfig)
 			if err != nil {
+				LOG_WARN("SFT_RECONNECT", "Failed to connect to %s: %v; retrying", storage.serverAddress, err)
 				storage.clientLock.Unlock()
-				return err
+				continue
 			}
 
 			client, err := sftp.NewClient(connection)
 			if err != nil {
+				LOG_WARN("SFT_RECONNECT", "Failed to create a new SFTP client to %s: %v; retrying", storage.serverAddress, err)
 				connection.Close()
 				storage.clientLock.Unlock()
-				return err
+				continue
 			}
 			storage.client = client
 			storage.clientLock.Unlock()
@@ -275,36 +272,19 @@ func (storage *SFTPStorage) UploadFile(threadIndex int, filePath string, content
 	fullPath := path.Join(storage.storageDir, filePath)
 
 	dirs := strings.Split(filePath, "/")
-	if len(dirs) > 1 {
+	fullDir := path.Dir(fullPath)
-		fullDir := path.Dir(fullPath)
+	return storage.retry(func() error {
-		err = storage.retry(func() error {
-			_, err := storage.getSFTPClient().Stat(fullDir)
-			return err
-		})
-		if err != nil {
-			// The error may be caused by a non-existent fullDir, or a broken connection.  In either case,
-			// we just assume it is the former because there isn't a way to tell which is the case.
-			for i := range dirs[1 : len(dirs)-1] {
-				subDir := path.Join(storage.storageDir, path.Join(dirs[0:i+2]...))
-				// We don't check the error; just keep going blindly but always store the last err
-				err = storage.getSFTPClient().Mkdir(subDir)
-			}
 
-			// If there is an error creating the dirs, we check fullDir one more time, because another thread
+		if len(dirs) > 1 {
-			// may happen to create the same fullDir ahead of this thread
+			_, err := storage.getSFTPClient().Stat(fullDir)
-			if err != nil {
+			if os.IsNotExist(err) {
-				err = storage.retry(func() error {
+				for i := range dirs[1 : len(dirs)-1] {
-					_, err := storage.getSFTPClient().Stat(fullDir)
+					subDir := path.Join(storage.storageDir, path.Join(dirs[0:i+2]...))
-					return err
+					// We don't check the error; just keep going blindly
-				})
+					storage.getSFTPClient().Mkdir(subDir)
-				if err != nil {
-					return err
 				}
 			}
 		}
-	}
-
-	return storage.retry(func() error {
 
 		letters := "abcdefghijklmnopqrstuvwxyz"
 		suffix := make([]byte, 8)

src/duplicacy_storage.go

@@ -531,7 +531,25 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 		accountID := GetPassword(preference, "b2_id", "Enter Backblaze account or application id:", true, resetPassword)
 		applicationKey := GetPassword(preference, "b2_key", "Enter corresponding Backblaze application key:", true, resetPassword)
 
-		b2Storage, err := CreateB2Storage(accountID, applicationKey, bucket, storageDir, threads)
+		b2Storage, err := CreateB2Storage(accountID, applicationKey, "", bucket, storageDir, threads)
+		if err != nil {
+			LOG_ERROR("STORAGE_CREATE", "Failed to load the Backblaze B2 storage at %s: %v", storageURL, err)
+			return nil
+		}
+		SavePassword(preference, "b2_id", accountID)
+		SavePassword(preference, "b2_key", applicationKey)
+		return b2Storage
+	} else if matched[1] == "b2-custom" {
+		b2customUrlRegex := regexp.MustCompile(`^b2-custom://([^/]+)/([^/]+)(/(.+))?`)
+		matched := b2customUrlRegex.FindStringSubmatch(storageURL)
+		downloadURL := "https://" + matched[1]
+		bucket := matched[2]
+		storageDir := matched[4]
+
+		accountID := GetPassword(preference, "b2_id", "Enter Backblaze account or application id:", true, resetPassword)
+		applicationKey := GetPassword(preference, "b2_key", "Enter corresponding Backblaze application key:", true, resetPassword)
+
+		b2Storage, err := CreateB2Storage(accountID, applicationKey, downloadURL, bucket, storageDir, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the Backblaze B2 storage at %s: %v", storageURL, err)
 			return nil

src/duplicacy_storage_test.go

@@ -109,7 +109,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
 	} else if testStorageName == "b2" {
-		storage, err := CreateB2Storage(config["account"], config["key"], config["bucket"], config["directory"], threads)
+		storage, err := CreateB2Storage(config["account"], config["key"], "", config["bucket"], config["directory"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
 	} else if testStorageName == "gcs-s3" {