jkl1337/duplicacy
1
0
Fork 0
mirror of https://github.com/jkl1337/duplicacy.git synced 2026-01-06 13:44:40 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into dry_run

Browse Source
This commit is contained in:
niknah
2017-09-11 20:12:09 +10:00
committed by GitHub
parent 457e518151 8808ad5c28
commit de2f7c447f
7 changed files with 194 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
src/duplicacy_backupmanager.go
View File

@@ -1491,14 +1491,27 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
return false
}
revisionMap := make(map[int]bool)
if snapshotID == "" && len(revisionsToBeCopied) > 0 {
LOG_ERROR("SNAPSHOT_ERROR", "You must specify the snapshot id when one or more revisions are specified.")
return false
}
revisionMap := make(map[string]map[int]bool)
_, found := revisionMap[snapshotID]
if !found {
revisionMap[snapshotID] = make(map[int]bool)
}
for _, revision := range revisionsToBeCopied {
revisionMap[revision] = true
revisionMap[snapshotID][revision] = true
}
var snapshots [] *Snapshot
var otherSnapshots [] *Snapshot
var snapshotIDs [] string
var err error
if snapshotID == "" {
snapshotIDs, err = manager.SnapshotManager.ListSnapshotIDs()
if err != nil {
@@ -1510,6 +1523,10 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
}
for _, id := range snapshotIDs {
_, found := revisionMap[id]
if !found {
revisionMap[id] = make(map[int]bool)
}
revisions, err := manager.SnapshotManager.ListSnapshotRevisions(id)
if err != nil {
LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions for snapshot %s: %v", id, err)
@@ -1518,9 +1535,14 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
for _, revision := range revisions {
if len(revisionsToBeCopied) > 0 {
if _, found := revisionMap[revision]; !found {
if _, found := revisionMap[id][revision]; found {
revisionMap[id][revision] = true
} else {
revisionMap[id][revision] = false
continue
}
} else {
revisionMap[id][revision] = true
}
snapshotPath := fmt.Sprintf("snapshots/%s/%d", id, revision)
@@ -1532,21 +1554,44 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
}
if exist {
LOG_INFO("SNAPSHOT_EXIST", "Snapshot %s at revision %d already exists in the destination storage",
LOG_INFO("SNAPSHOT_EXIST", "Snapshot %s at revision %d already exists at the destination storage",
id, revision)
revisionMap[id][revision] = false
continue
}
snapshot := manager.SnapshotManager.DownloadSnapshot(id, revision)
snapshots = append(snapshots, snapshot)
}
otherRevisions, err := otherManager.SnapshotManager.ListSnapshotRevisions(id)
if err != nil {
LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions at the destination for snapshot %s: %v", id, err)
return false
}
for _, otherRevision := range otherRevisions {
otherSnapshot := otherManager.SnapshotManager.DownloadSnapshot(id, otherRevision)
otherSnapshots = append(otherSnapshots, otherSnapshot)
}
}
if len(snapshots) == 0 {
LOG_INFO("SNAPSHOT_COPY", "Nothing to copy, all snapshot revisions exist at the destination.")
return true
}
chunks := make(map[string]bool)
for _, snapshot := range snapshots {
if revisionMap[snapshot.ID][snapshot.Revision] == false {
continue
}
LOG_TRACE("SNAPSHOT_COPY", "Copying snapshot %s at revision %d", snapshot.ID, snapshot.Revision)
for _, chunkHash := range snapshot.FileSequence {
chunks[chunkHash] = true
}
@@ -1572,44 +1617,92 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
}
}
for _, otherSnapshot := range otherSnapshots {
for _, chunkHash := range otherSnapshot.FileSequence {
if _, found := chunks[chunkHash]; found {
chunks[chunkHash] = false
}
}
for _, chunkHash := range otherSnapshot.ChunkSequence {
if _, found := chunks[chunkHash]; found {
chunks[chunkHash] = false
}
}
for _, chunkHash := range otherSnapshot.LengthSequence {
if _, found := chunks[chunkHash]; found {
chunks[chunkHash] = false
}
}
description := otherManager.SnapshotManager.DownloadSequence(otherSnapshot.ChunkSequence)
err := otherSnapshot.LoadChunks(description)
if err != nil {
LOG_ERROR("SNAPSHOT_CHUNK", "Failed to load chunks for destination snapshot %s at revision %d: %v",
otherSnapshot.ID, otherSnapshot.Revision, err)
return false
}
for _, chunkHash := range otherSnapshot.ChunkHashes {
if _, found := chunks[chunkHash]; found {
chunks[chunkHash] = false
}
}
}
chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, false, threads)
chunkUploader := CreateChunkUploader(otherManager.config, otherManager.storage, nil, threads,
func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
if skipped {
LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists in the destination", chunk.GetID(), chunkIndex, len(chunks))
LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists at the destination", chunk.GetID(), chunkIndex, len(chunks))
} else {
LOG_INFO("SNAPSHOT_COPY", "Copied chunk %s (%d/%d)", chunk.GetID(), chunkIndex, len(chunks))
LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) copied to the destination", chunk.GetID(), chunkIndex, len(chunks))
}
otherManager.config.PutChunk(chunk)
})
chunkUploader.Start()
totalCopied := 0
totalSkipped := 0
chunkIndex := 0
for chunkHash, _ := range chunks {
for chunkHash, needsCopy := range chunks {
chunkIndex++
chunkID := manager.config.GetChunkIDFromHash(chunkHash)
newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash)
LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID)
i := chunkDownloader.AddChunk(chunkHash)
chunk := chunkDownloader.WaitForChunk(i)
newChunk := otherManager.config.GetChunk()
newChunk.Reset(true)
newChunk.Write(chunk.GetBytes())
chunkUploader.StartChunk(newChunk, chunkIndex)
if needsCopy {
newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash)
LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID)
i := chunkDownloader.AddChunk(chunkHash)
chunk := chunkDownloader.WaitForChunk(i)
newChunk := otherManager.config.GetChunk()
newChunk.Reset(true)
newChunk.Write(chunk.GetBytes())
chunkUploader.StartChunk(newChunk, chunkIndex)
totalCopied++
} else {
LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists at the destination.", chunkID, chunkIndex, len(chunks))
totalSkipped++
}
}
chunkDownloader.Stop()
chunkUploader.Stop()
LOG_INFO("SNAPSHOT_COPY", "Total chunks copied = %d, skipped = %d.", totalCopied, totalSkipped)
for _, snapshot := range snapshots {
if !manager.config.dryRun {
otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", manager.snapshotID))
if revisionMap[snapshot.ID][snapshot.Revision] == false {
continue
}
}
otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", snapshot.ID))
description, _ := snapshot.MarshalJSON()
path := fmt.Sprintf("snapshots/%s/%d", manager.snapshotID, snapshot.Revision)
path := fmt.Sprintf("snapshots/%s/%d", snapshot.ID, snapshot.Revision)
otherManager.SnapshotManager.UploadFile(path, path, description)
LOG_INFO("SNAPSHOT_COPY", "Copied snapshot %s at revision %d", snapshot.ID, snapshot.Revision)
}

10
src/duplicacy_entry.go
View File

@@ -15,6 +15,7 @@ import (
"encoding/json"
"encoding/base64"
"strings"
"runtime"
)
@@ -488,7 +489,14 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns [] string
skippedFiles = append(skippedFiles, entry.Path)
continue
}
entry = CreateEntryFromFileInfo(stat, "")
newEntry := CreateEntryFromFileInfo(stat, "")
if runtime.GOOS == "windows" {
// On Windows, stat.Name() is the last component of the target, so we need to construct the correct
// path from f.Name(); note that a "/" is append assuming a symbolic link is always a directory
newEntry.Path = filepath.Join(normalizedPath, f.Name()) + "/"
}
entry = newEntry
}
}

28
src/duplicacy_s3storage.go
View File

@@ -5,6 +5,9 @@
package duplicacy
import (
"strings"
"reflect"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
@@ -227,15 +230,26 @@ func (storage *S3Storage) DownloadFile(threadIndex int, filePath string, chunk *
// UploadFile writes 'content' to the file at 'filePath'.
func (storage *S3Storage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
input := &s3.PutObjectInput {
Bucket: aws.String(storage.bucket),
Key: aws.String(storage.storageDir + filePath),
ACL: aws.String(s3.ObjectCannedACLPrivate),
Body: CreateRateLimitedReader(content, storage.UploadRateLimit / len(storage.bucket)),
ContentType: aws.String("application/duplicacy"),
attempts := 0
for {
input := &s3.PutObjectInput {
Bucket: aws.String(storage.bucket),
Key: aws.String(storage.storageDir + filePath),
ACL: aws.String(s3.ObjectCannedACLPrivate),
Body: CreateRateLimitedReader(content, storage.UploadRateLimit / len(storage.bucket)),
ContentType: aws.String("application/duplicacy"),
}
_, err = storage.client.PutObject(input)
if err == nil || attempts >= 3 || !strings.Contains(err.Error(), "XAmzContentSHA256Mismatch") {
return err
}
LOG_INFO("S3_RETRY", "Retrying on %s: %v", reflect.TypeOf(err), err)
attempts += 1
}
_, err = storage.client.PutObject(input)
return err
}

13
src/duplicacy_storage.go
View File

@@ -200,6 +200,9 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
username = username[:len(username) - 1]
}
// If ssh_key_file is set, skip password-based login
keyFile := GetPasswordFromPreference(preference, "ssh_key_file")
password := ""
passwordCallback := func() (string, error) {
LOG_DEBUG("SSH_PASSWORD", "Attempting password login")
@@ -219,7 +222,6 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
}
}
keyFile := ""
publicKeysCallback := func() ([]ssh.Signer, error) {
LOG_DEBUG("SSH_PUBLICKEY", "Attempting public key authentication")
@@ -273,10 +275,19 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
}
authMethods := [] ssh.AuthMethod {
}
passwordAuthMethods := [] ssh.AuthMethod {
ssh.PasswordCallback(passwordCallback),
ssh.KeyboardInteractive(keyboardInteractive),
}
keyFileAuthMethods := [] ssh.AuthMethod {
ssh.PublicKeysCallback(publicKeysCallback),
}
if keyFile != "" {
authMethods = append(keyFileAuthMethods, passwordAuthMethods...)
} else {
authMethods = append(passwordAuthMethods, keyFileAuthMethods...)
}
if RunInBackground {

35
src/duplicacy_utils.go
View File

@@ -118,10 +118,8 @@ func GenerateKeyFromPassword(password string) []byte {
return pbkdf2.Key([]byte(password), DEFAULT_KEY, 16384, 32, sha256.New)
}
// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input.
func GetPassword(preference Preference, passwordType string, prompt string,
showPassword bool, resetPassword bool) (string) {
// Get password from preference, env, but don't start any keyring request
func GetPasswordFromPreference(preference Preference, passwordType string) (string) {
passwordID := passwordType
if preference.Name != "default" {
passwordID = preference.Name + "_" + passwordID
@@ -135,11 +133,31 @@ func GetPassword(preference Preference, passwordType string, prompt string,
}
}
// If the password is stored in the preference, there is no need to include the storage name
// (i.e., preference.Name) in the key, so the key name should really be passwordType rather
// than passwordID; we're using passwordID here only for backward compatibility
if len(preference.Keys) > 0 && len(preference.Keys[passwordID]) > 0 {
LOG_DEBUG("PASSWORD_KEYCHAIN", "Reading %s from preferences", passwordID)
return preference.Keys[passwordID]
}
if len(preference.Keys) > 0 && len(preference.Keys[passwordType]) > 0 {
LOG_DEBUG("PASSWORD_KEYCHAIN", "Reading %s from preferences", passwordType)
return preference.Keys[passwordType]
}
return ""
}
// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input.
func GetPassword(preference Preference, passwordType string, prompt string,
showPassword bool, resetPassword bool) (string) {
passwordID := passwordType
password := GetPasswordFromPreference(preference,passwordType)
if password != "" {
return password
}
if resetPassword && !RunInBackground {
keyringSet(passwordID, "")
} else {
@@ -155,7 +173,7 @@ func GetPassword(preference Preference, passwordType string, prompt string,
}
password := ""
password = ""
fmt.Printf("%s", prompt)
if showPassword {
scanner := bufio.NewScanner(os.Stdin)
@@ -175,6 +193,7 @@ func GetPassword(preference Preference, passwordType string, prompt string,
// SavePassword saves the specified password in the keyring/keychain.
func SavePassword(preference Preference, passwordType string, password string) {
if password == "" || RunInBackground {
return
}
@@ -182,6 +201,12 @@ func SavePassword(preference Preference, passwordType string, password string) {
if preference.DoNotSavePassword {
return
}
// If the password is retrieved from env or preference, don't save it to keyring
if GetPasswordFromPreference(preference, passwordType) == password {
return
}
passwordID := passwordType
if preference.Name != "default" {
passwordID = preference.Name + "_" + passwordID