From 030cd274c2e09f7727016c5a75109159e20622a4 Mon Sep 17 00:00:00 2001 From: niknah Date: Mon, 4 Sep 2017 19:28:26 +1000 Subject: [PATCH 1/8] If we have a sftp key file in the environment/preferences, then don't attempt a password login to avoid bad login errors. --- src/duplicacy_storage.go | 11 ++++++++++- src/duplicacy_utils.go | 19 ++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index 819462b..d0e3638 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -199,6 +199,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor if username != "" { username = username[:len(username) - 1] } + keyFile := GetPasswordFromPreference(preference, "ssh_key_file") password := "" passwordCallback := func() (string, error) { @@ -219,7 +220,6 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } } - keyFile := "" publicKeysCallback := func() ([]ssh.Signer, error) { LOG_DEBUG("SSH_PUBLICKEY", "Attempting public key authentication") @@ -273,10 +273,19 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } authMethods := [] ssh.AuthMethod { + } + passwordAuthMethods := [] ssh.AuthMethod { ssh.PasswordCallback(passwordCallback), ssh.KeyboardInteractive(keyboardInteractive), + } + keyFileAuthMethods := [] ssh.AuthMethod { ssh.PublicKeysCallback(publicKeysCallback), } + if keyFile!="" { + authMethods = append(keyFileAuthMethods,passwordAuthMethods...) + } else { + authMethods = append(passwordAuthMethods,keyFileAuthMethods...) + } if RunInBackground { diff --git a/src/duplicacy_utils.go b/src/duplicacy_utils.go index feda7ac..757a756 100644 --- a/src/duplicacy_utils.go +++ b/src/duplicacy_utils.go @@ -118,10 +118,8 @@ func GenerateKeyFromPassword(password string) []byte { return pbkdf2.Key([]byte(password), DEFAULT_KEY, 16384, 32, sha256.New) } -// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input. -func GetPassword(preference Preference, passwordType string, prompt string, - showPassword bool, resetPassword bool) (string) { - +// Get password from preference, env, but don't start any keyring request +func GetPasswordFromPreference(preference Preference, passwordType string) (string) { passwordID := passwordType if preference.Name != "default" { passwordID = preference.Name + "_" + passwordID @@ -139,6 +137,17 @@ func GetPassword(preference Preference, passwordType string, prompt string, LOG_DEBUG("PASSWORD_KEYCHAIN", "Reading %s from preferences", passwordID) return preference.Keys[passwordID] } + return "" +} + +// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input. +func GetPassword(preference Preference, passwordType string, prompt string, + showPassword bool, resetPassword bool) (string) { + passwordID := passwordType + password := GetPasswordFromPreference(preference,passwordType) + if password != "" { + return password + } if resetPassword && !RunInBackground { keyringSet(passwordID, "") @@ -155,7 +164,7 @@ func GetPassword(preference Preference, passwordType string, prompt string, } - password := "" + password = "" fmt.Printf("%s", prompt) if showPassword { scanner := bufio.NewScanner(os.Stdin) From 34afc6f93c20c3e3f20b3ca751618df338d4b21c Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Tue, 5 Sep 2017 15:37:34 -0500 Subject: [PATCH 2/8] Update GUIDE.md Fix doc bug referenced in issue #151. --- GUIDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GUIDE.md b/GUIDE.md index 94029a8..e33b05c 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -440,10 +440,10 @@ For the *restore* command, the include/exclude patterns are specified as the com Duplicacy will attempt to retrieve in three ways the storage password and the storage-specific access tokens/keys. * If a secret vault service is available, Duplicacy will store passwords/keys entered by the user in such a secret vault and later retrieve them when needed. On Mac OS X it is Keychain, and on Linux it is gnome-keyring. On Windows the passwords/keys are encrypted and decrypted by the Data Protection API, and encrypted passwords/keys are stored in the file *.duplicacy/keyring*. However, if the -no-save-password option is specified for the storage, then Duplicacy will not save passwords this way. -* If an environment variable for a password is provided, Duplicacy will always take it. The table below shows the name of the environment variable for each kind of password. Note that if the storage is not the default one, the storage name will be included in the name of the environment variable. +* If an environment variable for a password is provided, Duplicacy will always take it. The table below shows the name of the environment variable for each kind of password. Note that if the storage is not the default one, the storage name will be included in the name of the environment variable (in uppercase). For example, if your storage name is b2, then the environment variable should be named DUPLICACY_B2_PASSWORD. * If a matching key and its value are saved to the preference file (.duplicacy/preferences) by the *set* command, the value will be used as the password. The last column in the table below lists the name of the preference key for each type of password. -| password type | environment variable (default storage) | environment variable (non-default storage) | key in preferences | +| password type | environment variable (default storage) | environment variable (non-default storage in uppercase) | key in preferences | |:----------------:|:----------------:|:----------------:|:----------------:| | storage password | DUPLICACY_PASSWORD | DUPLICACY_<STORAGENAME>_PASSWORD | password | | sftp password | DUPLICACY_SSH_PASSWORD | DUPLICACY_<STORAGENAME>_SSH_PASSWORD | ssh_password | From 612c5b774619e9d5d3a7f5fc151447e1a65355ca Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Wed, 6 Sep 2017 17:01:27 -0500 Subject: [PATCH 3/8] Use number of threads specified on copy command --- duplicacy/duplicacy_main.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index b00686b..3174cff 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -1020,12 +1020,17 @@ func copySnapshots(context *cli.Context) { os.Exit(ArgumentExitCode) } + threads := context.Int("threads") + if threads < 1 { + threads = 1 + } + repository, source := getRepositoryPreference(context, context.String("from")) runScript(context, source.Name, "pre") duplicacy.LOG_INFO("STORAGE_SET", "Source storage set to %s", source.StorageURL) - sourceStorage := duplicacy.CreateStorage(*source, false, 1) + sourceStorage := duplicacy.CreateStorage(*source, false, threads) if sourceStorage == nil { return } @@ -1055,7 +1060,7 @@ func copySnapshots(context *cli.Context) { duplicacy.LOG_INFO("STORAGE_SET", "Destination storage set to %s", destination.StorageURL) - destinationStorage := duplicacy.CreateStorage(*destination, false, 1) + destinationStorage := duplicacy.CreateStorage(*destination, false, threads) if destinationStorage == nil { return } @@ -1080,11 +1085,6 @@ func copySnapshots(context *cli.Context) { snapshotID = context.String("id") } - threads := context.Int("threads") - if threads < 1 { - threads = 1 - } - sourceManager.CopySnapshots(destinationManager, snapshotID, revisions, threads) runScript(context, source.Name, "post") } From b41e8a24a998690836075b6a78b9446c3b4d180b Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Sat, 2 Sep 2017 22:36:26 -0500 Subject: [PATCH 4/8] Skip chunks to copy if already on destination for issue #134 --- GUIDE.md | 4 +- duplicacy/duplicacy_main.go | 14 ++-- src/duplicacy_backupmanager.go | 131 ++++++++++++++++++++++++++++----- src/duplicacy_storage.go | 11 ++- src/duplicacy_utils.go | 19 +++-- 5 files changed, 145 insertions(+), 34 deletions(-) diff --git a/GUIDE.md b/GUIDE.md index 94029a8..e33b05c 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -440,10 +440,10 @@ For the *restore* command, the include/exclude patterns are specified as the com Duplicacy will attempt to retrieve in three ways the storage password and the storage-specific access tokens/keys. * If a secret vault service is available, Duplicacy will store passwords/keys entered by the user in such a secret vault and later retrieve them when needed. On Mac OS X it is Keychain, and on Linux it is gnome-keyring. On Windows the passwords/keys are encrypted and decrypted by the Data Protection API, and encrypted passwords/keys are stored in the file *.duplicacy/keyring*. However, if the -no-save-password option is specified for the storage, then Duplicacy will not save passwords this way. -* If an environment variable for a password is provided, Duplicacy will always take it. The table below shows the name of the environment variable for each kind of password. Note that if the storage is not the default one, the storage name will be included in the name of the environment variable. +* If an environment variable for a password is provided, Duplicacy will always take it. The table below shows the name of the environment variable for each kind of password. Note that if the storage is not the default one, the storage name will be included in the name of the environment variable (in uppercase). For example, if your storage name is b2, then the environment variable should be named DUPLICACY_B2_PASSWORD. * If a matching key and its value are saved to the preference file (.duplicacy/preferences) by the *set* command, the value will be used as the password. The last column in the table below lists the name of the preference key for each type of password. -| password type | environment variable (default storage) | environment variable (non-default storage) | key in preferences | +| password type | environment variable (default storage) | environment variable (non-default storage in uppercase) | key in preferences | |:----------------:|:----------------:|:----------------:|:----------------:| | storage password | DUPLICACY_PASSWORD | DUPLICACY_<STORAGENAME>_PASSWORD | password | | sftp password | DUPLICACY_SSH_PASSWORD | DUPLICACY_<STORAGENAME>_SSH_PASSWORD | ssh_password | diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index b00686b..3174cff 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -1020,12 +1020,17 @@ func copySnapshots(context *cli.Context) { os.Exit(ArgumentExitCode) } + threads := context.Int("threads") + if threads < 1 { + threads = 1 + } + repository, source := getRepositoryPreference(context, context.String("from")) runScript(context, source.Name, "pre") duplicacy.LOG_INFO("STORAGE_SET", "Source storage set to %s", source.StorageURL) - sourceStorage := duplicacy.CreateStorage(*source, false, 1) + sourceStorage := duplicacy.CreateStorage(*source, false, threads) if sourceStorage == nil { return } @@ -1055,7 +1060,7 @@ func copySnapshots(context *cli.Context) { duplicacy.LOG_INFO("STORAGE_SET", "Destination storage set to %s", destination.StorageURL) - destinationStorage := duplicacy.CreateStorage(*destination, false, 1) + destinationStorage := duplicacy.CreateStorage(*destination, false, threads) if destinationStorage == nil { return } @@ -1080,11 +1085,6 @@ func copySnapshots(context *cli.Context) { snapshotID = context.String("id") } - threads := context.Int("threads") - if threads < 1 { - threads = 1 - } - sourceManager.CopySnapshots(destinationManager, snapshotID, revisions, threads) runScript(context, source.Name, "post") } diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index 9090653..e23f8bc 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -1484,14 +1484,27 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho return false } - revisionMap := make(map[int]bool) + if snapshotID == "" && len(revisionsToBeCopied) > 0 { + LOG_ERROR("SNAPSHOT_ERROR", "You must specify the snapshot id when one or more revisions are specified.") + return false + } + + revisionMap := make(map[string]map[int]bool) + + _, found := revisionMap[snapshotID] + if !found { + revisionMap[snapshotID] = make(map[int]bool) + } + for _, revision := range revisionsToBeCopied { - revisionMap[revision] = true + revisionMap[snapshotID][revision] = true } var snapshots [] *Snapshot + var otherSnapshots [] *Snapshot var snapshotIDs [] string var err error + if snapshotID == "" { snapshotIDs, err = manager.SnapshotManager.ListSnapshotIDs() if err != nil { @@ -1503,6 +1516,10 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho } for _, id := range snapshotIDs { + _, found := revisionMap[id] + if !found { + revisionMap[id] = make(map[int]bool) + } revisions, err := manager.SnapshotManager.ListSnapshotRevisions(id) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions for snapshot %s: %v", id, err) @@ -1511,9 +1528,14 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho for _, revision := range revisions { if len(revisionsToBeCopied) > 0 { - if _, found := revisionMap[revision]; !found { + if _, found := revisionMap[id][revision]; found { + revisionMap[id][revision] = true + } else { + revisionMap[id][revision] = false continue } + } else { + revisionMap[id][revision] = true } snapshotPath := fmt.Sprintf("snapshots/%s/%d", id, revision) @@ -1525,21 +1547,44 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho } if exist { - LOG_INFO("SNAPSHOT_EXIST", "Snapshot %s at revision %d already exists in the destination storage", + LOG_INFO("SNAPSHOT_EXIST", "Snapshot %s at revision %d already exists at the destination storage", id, revision) + revisionMap[id][revision] = false continue } snapshot := manager.SnapshotManager.DownloadSnapshot(id, revision) snapshots = append(snapshots, snapshot) } + + otherRevisions, err := otherManager.SnapshotManager.ListSnapshotRevisions(id) + if err != nil { + LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions at the destination for snapshot %s: %v", id, err) + return false + } + + for _, otherRevision := range otherRevisions { + otherSnapshot := otherManager.SnapshotManager.DownloadSnapshot(id, otherRevision) + otherSnapshots = append(otherSnapshots, otherSnapshot) + } + + } + + if len(snapshots) == 0 { + LOG_INFO("SNAPSHOT_COPY", "Nothing to copy, all snapshot revisions exist at the destination.") + return true } chunks := make(map[string]bool) for _, snapshot := range snapshots { + if revisionMap[snapshot.ID][snapshot.Revision] == false { + continue + } + LOG_TRACE("SNAPSHOT_COPY", "Copying snapshot %s at revision %d", snapshot.ID, snapshot.Revision) + for _, chunkHash := range snapshot.FileSequence { chunks[chunkHash] = true } @@ -1565,42 +1610,90 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho } } + for _, otherSnapshot := range otherSnapshots { + + for _, chunkHash := range otherSnapshot.FileSequence { + if _, found := chunks[chunkHash]; found { + chunks[chunkHash] = false + } + } + + for _, chunkHash := range otherSnapshot.ChunkSequence { + if _, found := chunks[chunkHash]; found { + chunks[chunkHash] = false + } + } + + for _, chunkHash := range otherSnapshot.LengthSequence { + if _, found := chunks[chunkHash]; found { + chunks[chunkHash] = false + } + } + + description := otherManager.SnapshotManager.DownloadSequence(otherSnapshot.ChunkSequence) + err := otherSnapshot.LoadChunks(description) + if err != nil { + LOG_ERROR("SNAPSHOT_CHUNK", "Failed to load chunks for destination snapshot %s at revision %d: %v", + otherSnapshot.ID, otherSnapshot.Revision, err) + return false + } + + for _, chunkHash := range otherSnapshot.ChunkHashes { + if _, found := chunks[chunkHash]; found { + chunks[chunkHash] = false + } + } + } + chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, false, threads) chunkUploader := CreateChunkUploader(otherManager.config, otherManager.storage, nil, threads, func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) { if skipped { - LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists in the destination", chunk.GetID(), chunkIndex, len(chunks)) + LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists at the destination", chunk.GetID(), chunkIndex, len(chunks)) } else { - LOG_INFO("SNAPSHOT_COPY", "Copied chunk %s (%d/%d)", chunk.GetID(), chunkIndex, len(chunks)) + LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) copied to the destination", chunk.GetID(), chunkIndex, len(chunks)) } otherManager.config.PutChunk(chunk) }) + chunkUploader.Start() + totalCopied := 0 + totalSkipped := 0 chunkIndex := 0 - for chunkHash, _ := range chunks { + + for chunkHash, needsCopy := range chunks { chunkIndex++ chunkID := manager.config.GetChunkIDFromHash(chunkHash) - newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash) - - LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID) - - i := chunkDownloader.AddChunk(chunkHash) - chunk := chunkDownloader.WaitForChunk(i) - newChunk := otherManager.config.GetChunk() - newChunk.Reset(true) - newChunk.Write(chunk.GetBytes()) - chunkUploader.StartChunk(newChunk, chunkIndex) + if needsCopy { + newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash) + LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID) + i := chunkDownloader.AddChunk(chunkHash) + chunk := chunkDownloader.WaitForChunk(i) + newChunk := otherManager.config.GetChunk() + newChunk.Reset(true) + newChunk.Write(chunk.GetBytes()) + chunkUploader.StartChunk(newChunk, chunkIndex) + totalCopied++ + } else { + LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists at the destination.", chunkID, chunkIndex, len(chunks)) + totalSkipped++ + } } chunkDownloader.Stop() chunkUploader.Stop() + LOG_INFO("SNAPSHOT_COPY", "Total chunks copied = %d, skipped = %d.", totalCopied, totalSkipped) + for _, snapshot := range snapshots { - otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", manager.snapshotID)) + if revisionMap[snapshot.ID][snapshot.Revision] == false { + continue + } + otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", snapshot.ID)) description, _ := snapshot.MarshalJSON() - path := fmt.Sprintf("snapshots/%s/%d", manager.snapshotID, snapshot.Revision) + path := fmt.Sprintf("snapshots/%s/%d", snapshot.ID, snapshot.Revision) otherManager.SnapshotManager.UploadFile(path, path, description) LOG_INFO("SNAPSHOT_COPY", "Copied snapshot %s at revision %d", snapshot.ID, snapshot.Revision) } diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index 819462b..d0e3638 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -199,6 +199,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor if username != "" { username = username[:len(username) - 1] } + keyFile := GetPasswordFromPreference(preference, "ssh_key_file") password := "" passwordCallback := func() (string, error) { @@ -219,7 +220,6 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } } - keyFile := "" publicKeysCallback := func() ([]ssh.Signer, error) { LOG_DEBUG("SSH_PUBLICKEY", "Attempting public key authentication") @@ -273,10 +273,19 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } authMethods := [] ssh.AuthMethod { + } + passwordAuthMethods := [] ssh.AuthMethod { ssh.PasswordCallback(passwordCallback), ssh.KeyboardInteractive(keyboardInteractive), + } + keyFileAuthMethods := [] ssh.AuthMethod { ssh.PublicKeysCallback(publicKeysCallback), } + if keyFile!="" { + authMethods = append(keyFileAuthMethods,passwordAuthMethods...) + } else { + authMethods = append(passwordAuthMethods,keyFileAuthMethods...) + } if RunInBackground { diff --git a/src/duplicacy_utils.go b/src/duplicacy_utils.go index feda7ac..757a756 100644 --- a/src/duplicacy_utils.go +++ b/src/duplicacy_utils.go @@ -118,10 +118,8 @@ func GenerateKeyFromPassword(password string) []byte { return pbkdf2.Key([]byte(password), DEFAULT_KEY, 16384, 32, sha256.New) } -// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input. -func GetPassword(preference Preference, passwordType string, prompt string, - showPassword bool, resetPassword bool) (string) { - +// Get password from preference, env, but don't start any keyring request +func GetPasswordFromPreference(preference Preference, passwordType string) (string) { passwordID := passwordType if preference.Name != "default" { passwordID = preference.Name + "_" + passwordID @@ -139,6 +137,17 @@ func GetPassword(preference Preference, passwordType string, prompt string, LOG_DEBUG("PASSWORD_KEYCHAIN", "Reading %s from preferences", passwordID) return preference.Keys[passwordID] } + return "" +} + +// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input. +func GetPassword(preference Preference, passwordType string, prompt string, + showPassword bool, resetPassword bool) (string) { + passwordID := passwordType + password := GetPasswordFromPreference(preference,passwordType) + if password != "" { + return password + } if resetPassword && !RunInBackground { keyringSet(passwordID, "") @@ -155,7 +164,7 @@ func GetPassword(preference Preference, passwordType string, prompt string, } - password := "" + password = "" fmt.Printf("%s", prompt) if showPassword { scanner := bufio.NewScanner(os.Stdin) From b04ef67d26303363fc8ffc2ed29c2818d126d92a Mon Sep 17 00:00:00 2001 From: gilbertchen Date: Fri, 8 Sep 2017 11:57:46 -0400 Subject: [PATCH 5/8] Fixed a typo in GUIDE.md --- GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GUIDE.md b/GUIDE.md index e33b05c..285f169 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -290,7 +290,7 @@ chunks referenced by deleted snapshots but not any other snapshots. The `-exclusive` option will assume that no other clients are accessing the storage, effectively disabling the *two-step fossil collection* algorithm. With this option, the *prune* command will immediately remove unreferenced chunks. -The `-dryrun` option is used to test what changes the *prune* command would have done. It is guaranteed not to make any changes on the storage, not even creating the local fossil collection file. The following command checks if the chunk directory is clean (i.e., if there are any unreferenced chunks, temporary files, or anything else): +The `-dry-run` option is used to test what changes the *prune* command would have done. It is guaranteed not to make any changes on the storage, not even creating the local fossil collection file. The following command checks if the chunk directory is clean (i.e., if there are any unreferenced chunks, temporary files, or anything else): ``` $ duplicacy prune -d -exclusive -exhaustive # Prints out nothing if the chunk directory is clean From 68fb6d671ecb792d8920a34e64d84c88845140e5 Mon Sep 17 00:00:00 2001 From: Gilbert Chen Date: Fri, 8 Sep 2017 15:31:45 -0400 Subject: [PATCH 6/8] Fixed symbolic link handling on Windows --- src/duplicacy_entry.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/duplicacy_entry.go b/src/duplicacy_entry.go index 59cf32a..887bf09 100644 --- a/src/duplicacy_entry.go +++ b/src/duplicacy_entry.go @@ -15,6 +15,7 @@ import ( "encoding/json" "encoding/base64" "strings" + "runtime" ) @@ -488,7 +489,14 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns [] string skippedFiles = append(skippedFiles, entry.Path) continue } - entry = CreateEntryFromFileInfo(stat, "") + + newEntry := CreateEntryFromFileInfo(stat, "") + if runtime.GOOS == "windows" { + // On Windows, stat.Name() is the last component of the target, so we need to construct the correct + // path from f.Name(); note that a "/" is append assuming a symbolic link is always a directory + newEntry.Path = filepath.Join(normalizedPath, f.Name()) + "/" + } + entry = newEntry } } From 3f83890859ce131ce60f9c81a5a92cb82ade6979 Mon Sep 17 00:00:00 2001 From: Gilbert Chen Date: Fri, 8 Sep 2017 16:51:05 -0400 Subject: [PATCH 7/8] Don't save passwords from env/pref to keyring --- src/duplicacy_storage.go | 10 ++++++---- src/duplicacy_utils.go | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index d0e3638..13269da 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -199,7 +199,9 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor if username != "" { username = username[:len(username) - 1] } - keyFile := GetPasswordFromPreference(preference, "ssh_key_file") + + // If ssh_key_file is set, skip password-based login + keyFile := GetPasswordFromPreference(preference, "ssh_key_file") password := "" passwordCallback := func() (string, error) { @@ -281,10 +283,10 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor keyFileAuthMethods := [] ssh.AuthMethod { ssh.PublicKeysCallback(publicKeysCallback), } - if keyFile!="" { - authMethods = append(keyFileAuthMethods,passwordAuthMethods...) + if keyFile != "" { + authMethods = append(keyFileAuthMethods, passwordAuthMethods...) } else { - authMethods = append(passwordAuthMethods,keyFileAuthMethods...) + authMethods = append(passwordAuthMethods, keyFileAuthMethods...) } if RunInBackground { diff --git a/src/duplicacy_utils.go b/src/duplicacy_utils.go index 757a756..6abae55 100644 --- a/src/duplicacy_utils.go +++ b/src/duplicacy_utils.go @@ -133,10 +133,19 @@ func GetPasswordFromPreference(preference Preference, passwordType string) (stri } } + // If the password is stored in the preference, there is no need to include the storage name + // (i.e., preference.Name) in the key, so the key name should really be passwordType rather + // than passwordID; we're using passwordID here only for backward compatibility if len(preference.Keys) > 0 && len(preference.Keys[passwordID]) > 0 { LOG_DEBUG("PASSWORD_KEYCHAIN", "Reading %s from preferences", passwordID) return preference.Keys[passwordID] } + + if len(preference.Keys) > 0 && len(preference.Keys[passwordType]) > 0 { + LOG_DEBUG("PASSWORD_KEYCHAIN", "Reading %s from preferences", passwordType) + return preference.Keys[passwordType] + } + return "" } @@ -184,6 +193,7 @@ func GetPassword(preference Preference, passwordType string, prompt string, // SavePassword saves the specified password in the keyring/keychain. func SavePassword(preference Preference, passwordType string, password string) { + if password == "" || RunInBackground { return } @@ -191,6 +201,12 @@ func SavePassword(preference Preference, passwordType string, password string) { if preference.DoNotSavePassword { return } + + // If the password is retrieved from env or preference, don't save it to keyring + if GetPasswordFromPreference(preference, passwordType) == password { + return + } + passwordID := passwordType if preference.Name != "default" { passwordID = preference.Name + "_" + passwordID From 8808ad5c28ada2562ce6baf6504e7935a527f166 Mon Sep 17 00:00:00 2001 From: Gilbert Chen Date: Fri, 8 Sep 2017 19:46:27 -0400 Subject: [PATCH 8/8] Retry on XAmzContentSHA256Mismatch --- src/duplicacy_s3storage.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/duplicacy_s3storage.go b/src/duplicacy_s3storage.go index b328f2f..4c56cf0 100644 --- a/src/duplicacy_s3storage.go +++ b/src/duplicacy_s3storage.go @@ -5,6 +5,9 @@ package duplicacy import ( + "strings" + "reflect" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" @@ -227,15 +230,26 @@ func (storage *S3Storage) DownloadFile(threadIndex int, filePath string, chunk * // UploadFile writes 'content' to the file at 'filePath'. func (storage *S3Storage) UploadFile(threadIndex int, filePath string, content []byte) (err error) { - input := &s3.PutObjectInput { - Bucket: aws.String(storage.bucket), - Key: aws.String(storage.storageDir + filePath), - ACL: aws.String(s3.ObjectCannedACLPrivate), - Body: CreateRateLimitedReader(content, storage.UploadRateLimit / len(storage.bucket)), - ContentType: aws.String("application/duplicacy"), + attempts := 0 + + for { + input := &s3.PutObjectInput { + Bucket: aws.String(storage.bucket), + Key: aws.String(storage.storageDir + filePath), + ACL: aws.String(s3.ObjectCannedACLPrivate), + Body: CreateRateLimitedReader(content, storage.UploadRateLimit / len(storage.bucket)), + ContentType: aws.String("application/duplicacy"), + } + + _, err = storage.client.PutObject(input) + if err == nil || attempts >= 3 || !strings.Contains(err.Error(), "XAmzContentSHA256Mismatch") { + return err + } + + LOG_INFO("S3_RETRY", "Retrying on %s: %v", reflect.TypeOf(err), err) + attempts += 1 } - _, err = storage.client.PutObject(input) return err }