Skip chunks to copy if already on destination for issue #134

GUIDE.md

@@ -440,10 +440,10 @@ For the *restore* command, the include/exclude patterns are specified as the com
 Duplicacy will attempt to retrieve in three ways the storage password and the storage-specific access tokens/keys.
 
 * If a secret vault service is available, Duplicacy will store passwords/keys entered by the user in such a secret vault and later retrieve them when needed.  On Mac OS X it is Keychain, and on Linux it is gnome-keyring.  On Windows the passwords/keys are encrypted and decrypted by the Data Protection API, and encrypted passwords/keys are stored in the file *.duplicacy/keyring*.  However, if the -no-save-password option is specified for the storage, then Duplicacy will not save passwords this way.
-* If an environment variable for a password is provided, Duplicacy will always take it.  The table below shows the name of the environment variable for each kind of password.  Note that if the storage is not the default one, the storage name will be included in the name of the environment variable.
+* If an environment variable for a password is provided, Duplicacy will always take it.  The table below shows the name of the environment variable for each kind of password.  Note that if the storage is not the default one, the storage name will be included in the name of the environment variable (in uppercase). For example, if your storage name is b2, then the environment variable should be named DUPLICACY_B2_PASSWORD.
 * If a matching key and its value are saved to the preference file (.duplicacy/preferences) by the *set* command, the value will be used as the password.  The last column in the table below lists the name of the preference key for each type of password.
 
-| password type | environment variable (default storage) | environment variable (non-default storage) | key in preferences  |
+| password type | environment variable (default storage) | environment variable (non-default storage in uppercase) | key in preferences  |
 |:----------------:|:----------------:|:----------------:|:----------------:|
 | storage password | DUPLICACY_PASSWORD | DUPLICACY_<STORAGENAME>_PASSWORD | password |
 | sftp password | DUPLICACY_SSH_PASSWORD | DUPLICACY_<STORAGENAME>_SSH_PASSWORD | ssh_password |

duplicacy/duplicacy_main.go

@@ -1020,12 +1020,17 @@ func copySnapshots(context *cli.Context) {
         os.Exit(ArgumentExitCode)
     }
 
+    threads := context.Int("threads")
+    if threads < 1 {
+        threads = 1
+    }
+
     repository, source := getRepositoryPreference(context, context.String("from"))
 
     runScript(context, source.Name, "pre")
 
     duplicacy.LOG_INFO("STORAGE_SET", "Source storage set to %s", source.StorageURL)
-    sourceStorage := duplicacy.CreateStorage(*source, false, 1)
+    sourceStorage := duplicacy.CreateStorage(*source, false, threads)
     if sourceStorage == nil {
         return
     }
@@ -1055,7 +1060,7 @@ func copySnapshots(context *cli.Context) {
 
 
     duplicacy.LOG_INFO("STORAGE_SET", "Destination storage set to %s", destination.StorageURL)
-    destinationStorage := duplicacy.CreateStorage(*destination, false, 1)
+    destinationStorage := duplicacy.CreateStorage(*destination, false, threads)
     if destinationStorage == nil {
         return
     }
@@ -1080,11 +1085,6 @@ func copySnapshots(context *cli.Context) {
         snapshotID = context.String("id")
     }
 
-    threads := context.Int("threads")
-    if threads < 1 {
-        threads = 1
-    }
-
     sourceManager.CopySnapshots(destinationManager, snapshotID, revisions, threads)
     runScript(context, source.Name, "post")
 }

src/duplicacy_backupmanager.go

@@ -1484,14 +1484,27 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
         return false
     }
 
-    revisionMap := make(map[int]bool)
+    if snapshotID == "" && len(revisionsToBeCopied) > 0 {
+        LOG_ERROR("SNAPSHOT_ERROR", "You must specify the snapshot id when one or more revisions are specified.")
+        return false
+    }
+
+    revisionMap := make(map[string]map[int]bool)
+
+    _, found := revisionMap[snapshotID]
+    if !found {
+        revisionMap[snapshotID] = make(map[int]bool)
+    }
+
     for _, revision := range revisionsToBeCopied {
-        revisionMap[revision] = true
+        revisionMap[snapshotID][revision] = true
     }
 
     var snapshots [] *Snapshot
+    var otherSnapshots [] *Snapshot
     var snapshotIDs [] string
     var err error
+
     if snapshotID == "" {
         snapshotIDs, err = manager.SnapshotManager.ListSnapshotIDs()
         if err != nil {
@@ -1503,6 +1516,10 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
     }
 
     for _, id := range snapshotIDs {
+        _, found := revisionMap[id]
+        if !found {
+            revisionMap[id] = make(map[int]bool)
+        }
         revisions, err := manager.SnapshotManager.ListSnapshotRevisions(id)
         if err != nil {
             LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions for snapshot %s: %v", id, err)
@@ -1511,9 +1528,14 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 
         for _, revision := range revisions {
             if len(revisionsToBeCopied) > 0 {
-                if _, found := revisionMap[revision]; !found {
+                if _, found := revisionMap[id][revision]; found {
+                    revisionMap[id][revision] = true
+                } else {
+                    revisionMap[id][revision] = false
                     continue
                 }
+            } else {
+                revisionMap[id][revision] = true
             }
 
             snapshotPath := fmt.Sprintf("snapshots/%s/%d", id, revision)
@@ -1525,21 +1547,44 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
             }
 
             if exist {
-                LOG_INFO("SNAPSHOT_EXIST", "Snapshot %s at revision %d already exists in the destination storage",
+                LOG_INFO("SNAPSHOT_EXIST", "Snapshot %s at revision %d already exists at the destination storage",
                           id, revision)
+                revisionMap[id][revision] = false
                 continue
             }
 
             snapshot := manager.SnapshotManager.DownloadSnapshot(id, revision)
             snapshots = append(snapshots, snapshot)
         }
+
+        otherRevisions, err := otherManager.SnapshotManager.ListSnapshotRevisions(id)
+        if err != nil {
+            LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions at the destination for snapshot %s: %v", id, err)
+            return false
+        }
+
+        for _, otherRevision := range otherRevisions {
+            otherSnapshot := otherManager.SnapshotManager.DownloadSnapshot(id, otherRevision)
+            otherSnapshots = append(otherSnapshots, otherSnapshot)
+        }
+
+    }
+
+    if len(snapshots) == 0 {
+        LOG_INFO("SNAPSHOT_COPY", "Nothing to copy, all snapshot revisions exist at the destination.")
+        return true
     }
 
     chunks := make(map[string]bool)
 
     for _, snapshot := range snapshots {
 
+        if revisionMap[snapshot.ID][snapshot.Revision] == false {
+            continue
+        }
+
         LOG_TRACE("SNAPSHOT_COPY", "Copying snapshot %s at revision %d", snapshot.ID, snapshot.Revision)
+
         for _, chunkHash := range snapshot.FileSequence {
             chunks[chunkHash] = true
         }
@@ -1565,42 +1610,90 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
         }
     }
 
+    for _, otherSnapshot := range otherSnapshots {
+
+        for _, chunkHash := range otherSnapshot.FileSequence {
+            if  _, found := chunks[chunkHash]; found {
+                chunks[chunkHash] = false
+            }
+        }
+
+        for _, chunkHash := range otherSnapshot.ChunkSequence {
+            if _, found := chunks[chunkHash]; found {
+                chunks[chunkHash] = false
+            }
+        }
+
+        for _, chunkHash := range otherSnapshot.LengthSequence {
+            if _, found := chunks[chunkHash]; found {
+                chunks[chunkHash] = false
+            }
+        }
+
+        description := otherManager.SnapshotManager.DownloadSequence(otherSnapshot.ChunkSequence)
+        err := otherSnapshot.LoadChunks(description)
+        if err != nil {
+            LOG_ERROR("SNAPSHOT_CHUNK", "Failed to load chunks for destination snapshot %s at revision %d: %v",
+                      otherSnapshot.ID, otherSnapshot.Revision, err)
+            return false
+        }
+
+        for _, chunkHash := range otherSnapshot.ChunkHashes {
+            if _, found := chunks[chunkHash]; found {
+                chunks[chunkHash] = false
+            }
+        }
+    }
+
     chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, false, threads)
 
     chunkUploader := CreateChunkUploader(otherManager.config, otherManager.storage, nil, threads,
         func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
             if skipped {
-                LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists in the destination", chunk.GetID(), chunkIndex, len(chunks))
+                LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists at the destination", chunk.GetID(), chunkIndex, len(chunks))
             } else {
-                LOG_INFO("SNAPSHOT_COPY", "Copied chunk %s (%d/%d)", chunk.GetID(), chunkIndex, len(chunks))
+                LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) copied to the destination", chunk.GetID(), chunkIndex, len(chunks))
             }
             otherManager.config.PutChunk(chunk)
         })
+
     chunkUploader.Start()
 
+    totalCopied := 0
+    totalSkipped := 0
     chunkIndex := 0
-    for chunkHash, _ := range chunks {
+
+    for chunkHash, needsCopy := range chunks {
         chunkIndex++
         chunkID := manager.config.GetChunkIDFromHash(chunkHash)
-        newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash)
+        if needsCopy {
-
+            newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash)
-        LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID)
+            LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID)
-
+            i := chunkDownloader.AddChunk(chunkHash)
-        i := chunkDownloader.AddChunk(chunkHash)
+            chunk := chunkDownloader.WaitForChunk(i)
-        chunk := chunkDownloader.WaitForChunk(i)
+            newChunk := otherManager.config.GetChunk()
-        newChunk := otherManager.config.GetChunk()
+            newChunk.Reset(true)
-        newChunk.Reset(true)
+            newChunk.Write(chunk.GetBytes())
-        newChunk.Write(chunk.GetBytes())
+            chunkUploader.StartChunk(newChunk, chunkIndex)
-        chunkUploader.StartChunk(newChunk, chunkIndex)
+            totalCopied++
+        } else {
+            LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists at the destination.", chunkID, chunkIndex, len(chunks))
+            totalSkipped++
+        }
     }
 
     chunkDownloader.Stop()
     chunkUploader.Stop()
 
+    LOG_INFO("SNAPSHOT_COPY", "Total chunks copied = %d, skipped = %d.", totalCopied, totalSkipped)
+
     for _, snapshot := range snapshots {
-        otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", manager.snapshotID))
+        if revisionMap[snapshot.ID][snapshot.Revision] == false {
+            continue
+        }
+        otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", snapshot.ID))
         description, _ := snapshot.MarshalJSON()
-        path := fmt.Sprintf("snapshots/%s/%d", manager.snapshotID, snapshot.Revision)
+        path := fmt.Sprintf("snapshots/%s/%d", snapshot.ID, snapshot.Revision)
         otherManager.SnapshotManager.UploadFile(path, path, description)
         LOG_INFO("SNAPSHOT_COPY", "Copied snapshot %s at revision %d", snapshot.ID, snapshot.Revision)
     }

src/duplicacy_storage.go

@@ -199,6 +199,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
         if username != "" {
             username = username[:len(username) - 1]
         }
+	keyFile := GetPasswordFromPreference(preference, "ssh_key_file")
 
         password := ""
         passwordCallback := func() (string, error) {
@@ -219,7 +220,6 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
             }
         }
 
-        keyFile := ""
         publicKeysCallback := func() ([]ssh.Signer, error) {
             LOG_DEBUG("SSH_PUBLICKEY", "Attempting public key authentication")
 
@@ -273,10 +273,19 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
         }
 
         authMethods := [] ssh.AuthMethod {
+        }
+        passwordAuthMethods := [] ssh.AuthMethod {
             ssh.PasswordCallback(passwordCallback),
             ssh.KeyboardInteractive(keyboardInteractive),
+        }
+        keyFileAuthMethods := [] ssh.AuthMethod {
             ssh.PublicKeysCallback(publicKeysCallback),
         }
+	if keyFile!="" {
+	    authMethods = append(keyFileAuthMethods,passwordAuthMethods...)
+        } else {
+	    authMethods = append(passwordAuthMethods,keyFileAuthMethods...)
+        }
 
         if RunInBackground {
 

src/duplicacy_utils.go

@@ -118,10 +118,8 @@ func GenerateKeyFromPassword(password string) []byte {
     return pbkdf2.Key([]byte(password), DEFAULT_KEY, 16384, 32, sha256.New)
 }
 
-// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input.
+// Get password from preference, env, but don't start any keyring request
-func GetPassword(preference Preference, passwordType string, prompt string,
+func GetPasswordFromPreference(preference Preference, passwordType string) (string) {
-                 showPassword bool, resetPassword bool) (string) {
-
     passwordID := passwordType
     if preference.Name != "default" {
         passwordID = preference.Name + "_" + passwordID
@@ -139,6 +137,17 @@ func GetPassword(preference Preference, passwordType string, prompt string,
         LOG_DEBUG("PASSWORD_KEYCHAIN", "Reading %s from preferences", passwordID)
         return preference.Keys[passwordID]
     }
+    return ""
+}
+
+// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input.
+func GetPassword(preference Preference, passwordType string, prompt string,
+                 showPassword bool, resetPassword bool) (string) {
+    passwordID := passwordType
+    password := GetPasswordFromPreference(preference,passwordType)
+    if password != "" {
+        return password
+    }
 
     if resetPassword && !RunInBackground {
         keyringSet(passwordID, "")
@@ -155,7 +164,7 @@ func GetPassword(preference Preference, passwordType string, prompt string,
 
     }
 
-    password := ""
+    password = ""
     fmt.Printf("%s", prompt)
     if showPassword {
         scanner := bufio.NewScanner(os.Stdin)