Merge branch 'master' into dry_run

GUIDE.md

@@ -290,7 +290,7 @@ chunks referenced by deleted snapshots but not any other snapshots.
 
 The `-exclusive` option will assume that no other clients are accessing the storage, effectively disabling the *two-step fossil collection* algorithm.  With this option, the *prune* command will immediately remove unreferenced chunks.
 
-The `-dryrun` option is used to test what changes the *prune* command would have done.  It is guaranteed not to make any changes on the storage, not even creating the local fossil collection file.  The following command checks if the chunk directory is clean (i.e., if there are any unreferenced chunks, temporary files, or anything else):
+The `-dry-run` option is used to test what changes the *prune* command would have done.  It is guaranteed not to make any changes on the storage, not even creating the local fossil collection file.  The following command checks if the chunk directory is clean (i.e., if there are any unreferenced chunks, temporary files, or anything else):
 
 ```
 $ duplicacy prune -d -exclusive -exhaustive    #  Prints out nothing if the chunk directory is clean
@@ -440,10 +440,10 @@ For the *restore* command, the include/exclude patterns are specified as the com
 Duplicacy will attempt to retrieve in three ways the storage password and the storage-specific access tokens/keys.
 
 * If a secret vault service is available, Duplicacy will store passwords/keys entered by the user in such a secret vault and later retrieve them when needed.  On Mac OS X it is Keychain, and on Linux it is gnome-keyring.  On Windows the passwords/keys are encrypted and decrypted by the Data Protection API, and encrypted passwords/keys are stored in the file *.duplicacy/keyring*.  However, if the -no-save-password option is specified for the storage, then Duplicacy will not save passwords this way.
-* If an environment variable for a password is provided, Duplicacy will always take it.  The table below shows the name of the environment variable for each kind of password.  Note that if the storage is not the default one, the storage name will be included in the name of the environment variable.
+* If an environment variable for a password is provided, Duplicacy will always take it.  The table below shows the name of the environment variable for each kind of password.  Note that if the storage is not the default one, the storage name will be included in the name of the environment variable (in uppercase). For example, if your storage name is b2, then the environment variable should be named DUPLICACY_B2_PASSWORD.
 * If a matching key and its value are saved to the preference file (.duplicacy/preferences) by the *set* command, the value will be used as the password.  The last column in the table below lists the name of the preference key for each type of password.
 
-| password type | environment variable (default storage) | environment variable (non-default storage) | key in preferences  |
+| password type | environment variable (default storage) | environment variable (non-default storage in uppercase) | key in preferences  |
 |:----------------:|:----------------:|:----------------:|:----------------:|
 | storage password | DUPLICACY_PASSWORD | DUPLICACY_<STORAGENAME>_PASSWORD | password |
 | sftp password | DUPLICACY_SSH_PASSWORD | DUPLICACY_<STORAGENAME>_SSH_PASSWORD | ssh_password |

duplicacy/duplicacy_main.go

@@ -1022,12 +1022,17 @@ func copySnapshots(context *cli.Context) {
         os.Exit(ArgumentExitCode)
     }
 
+    threads := context.Int("threads")
+    if threads < 1 {
+        threads = 1
+    }
+
     repository, source := getRepositoryPreference(context, context.String("from"))
 
     runScript(context, source.Name, "pre")
 
     duplicacy.LOG_INFO("STORAGE_SET", "Source storage set to %s", source.StorageURL)
-    sourceStorage := duplicacy.CreateStorage(*source, false, 1)
+    sourceStorage := duplicacy.CreateStorage(*source, false, threads)
     if sourceStorage == nil {
         return
     }
@@ -1057,7 +1062,7 @@ func copySnapshots(context *cli.Context) {
 
 
     duplicacy.LOG_INFO("STORAGE_SET", "Destination storage set to %s", destination.StorageURL)
-    destinationStorage := duplicacy.CreateStorage(*destination, false, 1)
+    destinationStorage := duplicacy.CreateStorage(*destination, false, threads)
     if destinationStorage == nil {
         return
     }
@@ -1082,11 +1087,6 @@ func copySnapshots(context *cli.Context) {
         snapshotID = context.String("id")
     }
 
-    threads := context.Int("threads")
-    if threads < 1 {
-        threads = 1
-    }
-
     sourceManager.CopySnapshots(destinationManager, snapshotID, revisions, threads)
     runScript(context, source.Name, "post")
 }

src/duplicacy_backupmanager.go

@@ -1491,14 +1491,27 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
         return false
     }
 
-    revisionMap := make(map[int]bool)
+    if snapshotID == "" && len(revisionsToBeCopied) > 0 {
+        LOG_ERROR("SNAPSHOT_ERROR", "You must specify the snapshot id when one or more revisions are specified.")
+        return false
+    }
+
+    revisionMap := make(map[string]map[int]bool)
+
+    _, found := revisionMap[snapshotID]
+    if !found {
+        revisionMap[snapshotID] = make(map[int]bool)
+    }
+
     for _, revision := range revisionsToBeCopied {
-        revisionMap[revision] = true
+        revisionMap[snapshotID][revision] = true
     }
 
     var snapshots [] *Snapshot
+    var otherSnapshots [] *Snapshot
     var snapshotIDs [] string
     var err error
+
     if snapshotID == "" {
         snapshotIDs, err = manager.SnapshotManager.ListSnapshotIDs()
         if err != nil {
@@ -1510,6 +1523,10 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
     }
 
     for _, id := range snapshotIDs {
+        _, found := revisionMap[id]
+        if !found {
+            revisionMap[id] = make(map[int]bool)
+        }
         revisions, err := manager.SnapshotManager.ListSnapshotRevisions(id)
         if err != nil {
             LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions for snapshot %s: %v", id, err)
@@ -1518,9 +1535,14 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 
         for _, revision := range revisions {
             if len(revisionsToBeCopied) > 0 {
-                if _, found := revisionMap[revision]; !found {
+                if _, found := revisionMap[id][revision]; found {
+                    revisionMap[id][revision] = true
+                } else {
+                    revisionMap[id][revision] = false
                     continue
                 }
+            } else {
+                revisionMap[id][revision] = true
             }
 
             snapshotPath := fmt.Sprintf("snapshots/%s/%d", id, revision)
@@ -1532,21 +1554,44 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
             }
 
             if exist {
-                LOG_INFO("SNAPSHOT_EXIST", "Snapshot %s at revision %d already exists in the destination storage",
+                LOG_INFO("SNAPSHOT_EXIST", "Snapshot %s at revision %d already exists at the destination storage",
                           id, revision)
+                revisionMap[id][revision] = false
                 continue
             }
 
             snapshot := manager.SnapshotManager.DownloadSnapshot(id, revision)
             snapshots = append(snapshots, snapshot)
         }
+
+        otherRevisions, err := otherManager.SnapshotManager.ListSnapshotRevisions(id)
+        if err != nil {
+            LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions at the destination for snapshot %s: %v", id, err)
+            return false
+        }
+
+        for _, otherRevision := range otherRevisions {
+            otherSnapshot := otherManager.SnapshotManager.DownloadSnapshot(id, otherRevision)
+            otherSnapshots = append(otherSnapshots, otherSnapshot)
+        }
+
+    }
+
+    if len(snapshots) == 0 {
+        LOG_INFO("SNAPSHOT_COPY", "Nothing to copy, all snapshot revisions exist at the destination.")
+        return true
     }
 
     chunks := make(map[string]bool)
 
     for _, snapshot := range snapshots {
 
+        if revisionMap[snapshot.ID][snapshot.Revision] == false {
+            continue
+        }
+
         LOG_TRACE("SNAPSHOT_COPY", "Copying snapshot %s at revision %d", snapshot.ID, snapshot.Revision)
+
         for _, chunkHash := range snapshot.FileSequence {
             chunks[chunkHash] = true
         }
@@ -1572,44 +1617,92 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
         }
     }
 
+    for _, otherSnapshot := range otherSnapshots {
+
+        for _, chunkHash := range otherSnapshot.FileSequence {
+            if  _, found := chunks[chunkHash]; found {
+                chunks[chunkHash] = false
+            }
+        }
+
+        for _, chunkHash := range otherSnapshot.ChunkSequence {
+            if _, found := chunks[chunkHash]; found {
+                chunks[chunkHash] = false
+            }
+        }
+
+        for _, chunkHash := range otherSnapshot.LengthSequence {
+            if _, found := chunks[chunkHash]; found {
+                chunks[chunkHash] = false
+            }
+        }
+
+        description := otherManager.SnapshotManager.DownloadSequence(otherSnapshot.ChunkSequence)
+        err := otherSnapshot.LoadChunks(description)
+        if err != nil {
+            LOG_ERROR("SNAPSHOT_CHUNK", "Failed to load chunks for destination snapshot %s at revision %d: %v",
+                      otherSnapshot.ID, otherSnapshot.Revision, err)
+            return false
+        }
+
+        for _, chunkHash := range otherSnapshot.ChunkHashes {
+            if _, found := chunks[chunkHash]; found {
+                chunks[chunkHash] = false
+            }
+        }
+    }
+
     chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, false, threads)
 
     chunkUploader := CreateChunkUploader(otherManager.config, otherManager.storage, nil, threads,
         func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
             if skipped {
-                LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists in the destination", chunk.GetID(), chunkIndex, len(chunks))
+                LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists at the destination", chunk.GetID(), chunkIndex, len(chunks))
             } else {
-                LOG_INFO("SNAPSHOT_COPY", "Copied chunk %s (%d/%d)", chunk.GetID(), chunkIndex, len(chunks))
+                LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) copied to the destination", chunk.GetID(), chunkIndex, len(chunks))
             }
             otherManager.config.PutChunk(chunk)
         })
+
     chunkUploader.Start()
 
+    totalCopied := 0
+    totalSkipped := 0
     chunkIndex := 0
-    for chunkHash, _ := range chunks {
+
+    for chunkHash, needsCopy := range chunks {
         chunkIndex++
         chunkID := manager.config.GetChunkIDFromHash(chunkHash)
-        newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash)
-
-        LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID)
-
-        i := chunkDownloader.AddChunk(chunkHash)
-        chunk := chunkDownloader.WaitForChunk(i)
-        newChunk := otherManager.config.GetChunk()
-        newChunk.Reset(true)
-        newChunk.Write(chunk.GetBytes())
-        chunkUploader.StartChunk(newChunk, chunkIndex)
+        if needsCopy {
+            newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash)
+            LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID)
+            i := chunkDownloader.AddChunk(chunkHash)
+            chunk := chunkDownloader.WaitForChunk(i)
+            newChunk := otherManager.config.GetChunk()
+            newChunk.Reset(true)
+            newChunk.Write(chunk.GetBytes())
+            chunkUploader.StartChunk(newChunk, chunkIndex)
+            totalCopied++
+        } else {
+            LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists at the destination.", chunkID, chunkIndex, len(chunks))
+            totalSkipped++
+        }
     }
 
     chunkDownloader.Stop()
     chunkUploader.Stop()
 
+    LOG_INFO("SNAPSHOT_COPY", "Total chunks copied = %d, skipped = %d.", totalCopied, totalSkipped)
+
     for _, snapshot := range snapshots {
         if !manager.config.dryRun {
-            otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", manager.snapshotID))
+            if revisionMap[snapshot.ID][snapshot.Revision] == false {
+              continue
+          }
         }
+        otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", snapshot.ID))
         description, _ := snapshot.MarshalJSON()
-        path := fmt.Sprintf("snapshots/%s/%d", manager.snapshotID, snapshot.Revision)
+        path := fmt.Sprintf("snapshots/%s/%d", snapshot.ID, snapshot.Revision)
         otherManager.SnapshotManager.UploadFile(path, path, description)
         LOG_INFO("SNAPSHOT_COPY", "Copied snapshot %s at revision %d", snapshot.ID, snapshot.Revision)
     }

src/duplicacy_entry.go

@@ -15,6 +15,7 @@ import (
     "encoding/json"
     "encoding/base64"
     "strings"
+    "runtime"
 
 )
 
@@ -488,7 +489,14 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns [] string
                     skippedFiles = append(skippedFiles, entry.Path)
                     continue
                 }
-                entry = CreateEntryFromFileInfo(stat, "")
+
+                newEntry := CreateEntryFromFileInfo(stat, "")
+                if runtime.GOOS == "windows" {
+                    // On Windows, stat.Name() is the last component of the target, so we need to construct the correct
+                    // path from f.Name(); note that a "/" is append assuming a symbolic link is always a directory
+                    newEntry.Path = filepath.Join(normalizedPath, f.Name()) + "/"
+                }
+                entry = newEntry
             }
         }
 

src/duplicacy_s3storage.go

@@ -5,6 +5,9 @@
 package duplicacy
 
 import (
+    "strings"
+    "reflect"
+
     "github.com/aws/aws-sdk-go/aws"
     "github.com/aws/aws-sdk-go/aws/awserr"
     "github.com/aws/aws-sdk-go/aws/credentials"
@@ -227,15 +230,26 @@ func (storage *S3Storage) DownloadFile(threadIndex int, filePath string, chunk *
 // UploadFile writes 'content' to the file at 'filePath'.
 func (storage *S3Storage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
 
-    input := &s3.PutObjectInput {
-        Bucket: aws.String(storage.bucket),
-        Key: aws.String(storage.storageDir + filePath),
-        ACL: aws.String(s3.ObjectCannedACLPrivate),
-        Body: CreateRateLimitedReader(content, storage.UploadRateLimit / len(storage.bucket)),
-        ContentType: aws.String("application/duplicacy"),
+    attempts := 0
+
+    for {
+        input := &s3.PutObjectInput {
+            Bucket: aws.String(storage.bucket),
+            Key: aws.String(storage.storageDir + filePath),
+            ACL: aws.String(s3.ObjectCannedACLPrivate),
+            Body: CreateRateLimitedReader(content, storage.UploadRateLimit / len(storage.bucket)),
+            ContentType: aws.String("application/duplicacy"),
+        }
+
+        _, err = storage.client.PutObject(input)
+        if err == nil || attempts >= 3 || !strings.Contains(err.Error(), "XAmzContentSHA256Mismatch")  {
+            return err
+        }
+
+        LOG_INFO("S3_RETRY", "Retrying on %s: %v", reflect.TypeOf(err), err)
+        attempts += 1
     }
     
-    _, err = storage.client.PutObject(input)
     return err
 }
 

src/duplicacy_storage.go

@@ -200,6 +200,9 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
             username = username[:len(username) - 1]
         }
 
+        // If ssh_key_file is set, skip password-based login
+        keyFile := GetPasswordFromPreference(preference, "ssh_key_file")
+
         password := ""
         passwordCallback := func() (string, error) {
             LOG_DEBUG("SSH_PASSWORD", "Attempting password login")
@@ -219,7 +222,6 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
             }
         }
 
-        keyFile := ""
         publicKeysCallback := func() ([]ssh.Signer, error) {
             LOG_DEBUG("SSH_PUBLICKEY", "Attempting public key authentication")
 
@@ -273,10 +275,19 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
         }
 
         authMethods := [] ssh.AuthMethod {
+        }
+        passwordAuthMethods := [] ssh.AuthMethod {
             ssh.PasswordCallback(passwordCallback),
             ssh.KeyboardInteractive(keyboardInteractive),
+        }
+        keyFileAuthMethods := [] ssh.AuthMethod {
             ssh.PublicKeysCallback(publicKeysCallback),
         }
+        if keyFile != "" {
+            authMethods = append(keyFileAuthMethods, passwordAuthMethods...)
+        } else {
+            authMethods = append(passwordAuthMethods, keyFileAuthMethods...)
+        }
 
         if RunInBackground {
 

src/duplicacy_utils.go

@@ -118,10 +118,8 @@ func GenerateKeyFromPassword(password string) []byte {
     return pbkdf2.Key([]byte(password), DEFAULT_KEY, 16384, 32, sha256.New)
 }
 
-// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input.
-func GetPassword(preference Preference, passwordType string, prompt string,
-                 showPassword bool, resetPassword bool) (string) {
-
+// Get password from preference, env, but don't start any keyring request
+func GetPasswordFromPreference(preference Preference, passwordType string) (string) {
     passwordID := passwordType
     if preference.Name != "default" {
         passwordID = preference.Name + "_" + passwordID
@@ -135,11 +133,31 @@ func GetPassword(preference Preference, passwordType string, prompt string,
         }
     }
 
+    // If the password is stored in the preference, there is no need to include the storage name
+    // (i.e., preference.Name) in the key, so the key name should really be passwordType rather
+    // than passwordID; we're using passwordID here only for backward compatibility
     if len(preference.Keys) > 0 && len(preference.Keys[passwordID]) > 0 {
         LOG_DEBUG("PASSWORD_KEYCHAIN", "Reading %s from preferences", passwordID)
         return preference.Keys[passwordID]
     }
 
+    if len(preference.Keys) > 0 && len(preference.Keys[passwordType]) > 0 {
+        LOG_DEBUG("PASSWORD_KEYCHAIN", "Reading %s from preferences", passwordType)
+        return preference.Keys[passwordType]
+    }
+
+    return ""
+}
+
+// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input.
+func GetPassword(preference Preference, passwordType string, prompt string,
+                 showPassword bool, resetPassword bool) (string) {
+    passwordID := passwordType
+    password := GetPasswordFromPreference(preference,passwordType)
+    if password != "" {
+        return password
+    }
+
     if resetPassword && !RunInBackground {
         keyringSet(passwordID, "")
     } else {
@@ -155,7 +173,7 @@ func GetPassword(preference Preference, passwordType string, prompt string,
 
     }
 
-    password := ""
+    password = ""
     fmt.Printf("%s", prompt)
     if showPassword {
         scanner := bufio.NewScanner(os.Stdin)
@@ -175,6 +193,7 @@ func GetPassword(preference Preference, passwordType string, prompt string,
 
 // SavePassword saves the specified password in the keyring/keychain.
 func SavePassword(preference Preference, passwordType string, password string) {
+
     if password == "" || RunInBackground {
         return
     }
@@ -182,6 +201,12 @@ func SavePassword(preference Preference, passwordType string, password string) {
     if preference.DoNotSavePassword {
         return
     }
+
+    // If the password is retrieved from env or preference, don't save it to keyring
+    if GetPasswordFromPreference(preference, passwordType) == password {
+        return
+    }
+
     passwordID := passwordType
     if preference.Name != "default" {
         passwordID = preference.Name + "_" + passwordID