Bump version to 2.7.2

Only letter, numbers, dashes, and underscores are allowed.

Gopkg.lock

@@ -52,7 +52,7 @@
 [[projects]]
   name = "github.com/gilbertchen/go-dropbox"
   packages = ["."]
-  revision = "0baa9015ac2547d8b69b2e88c709aa90cfb8fbc1"
+  revision = "2233fa1dd846b3a3e8060b6c1ea12883deb9d288"
 
 [[projects]]
   name = "github.com/gilbertchen/go-ole"
@@ -173,6 +173,12 @@
   revision = "5616182052227b951e76d9c9b79a616c608bd91b"
   version = "v1.11.0"
 
+[[projects]]
+  name = "github.com/pkg/xattr"
+  packages = ["."]
+  revision = "dd870b5cfebab49617ea0c1da6176474e8a52bf4"
+  version = "v0.4.1"
+
 [[projects]]
   name = "github.com/satori/go.uuid"
   packages = ["."]
@@ -265,6 +271,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "0e6ea2be64dedc36cb9192f1d410917ea72896302011e55b6df5e4c00c1c2f1c"
+  inputs-digest = "e46f7c2dac527af6d5a0d47f1444421a6738d28252eb5d6084fc1c65f2b41bd8"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -47,7 +47,7 @@
 
 [[constraint]]
   name = "github.com/gilbertchen/go-dropbox"
-  revision = "0baa9015ac2547d8b69b2e88c709aa90cfb8fbc1"
+  revision = "2233fa1dd846b3a3e8060b6c1ea12883deb9d288"
 
 [[constraint]]
   name = "github.com/gilbertchen/go-ole"

duplicacy/duplicacy_main.go

@@ -274,6 +274,13 @@ func configRepository(context *cli.Context, init bool) {
 		}
 	}
 
+	snapshotIDRegex := regexp.MustCompile(`^[A-Za-z0-9_\-]+$`)
+	matched := snapshotIDRegex.FindStringSubmatch(snapshotID)
+	if matched == nil {
+		duplicacy.LOG_ERROR("PREFERENCE_INVALID", "'%s' is an invalid snapshot id", snapshotID)
+		return
+	}
+
 	var repository string
 	var err error
 
@@ -2179,7 +2186,7 @@ func main() {
 	app.Name = "duplicacy"
 	app.HelpName = "duplicacy"
 	app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
-	app.Version = "2.7.1" + " (" + GitCommit + ")"
+	app.Version = "2.7.2" + " (" + GitCommit + ")"
 
 	// If the program is interrupted, call the RunAtError function.
 	c := make(chan os.Signal, 1)

src/duplicacy_backupmanager.go

@@ -1377,7 +1377,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			}
 
 			// fileHash != entry.Hash, warn/error depending on -overwrite option
-			if !overwrite {
+			if !overwrite && !isNewFile {
 				LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
 							"File %s already exists.  Please specify the -overwrite option to overwrite", entry.Path)
 				return false, fmt.Errorf("file exists")

src/duplicacy_shadowcopy_darwin.go

@@ -14,7 +14,6 @@ import (
 	"os"
 	"os/exec"
 	"regexp"
-	"strings"
 	"syscall"
 	"time"
 )
@@ -77,19 +76,19 @@ func DeleteShadowCopy() {
 
 	err := exec.Command("/sbin/umount", "-f", snapshotPath).Run()
 	if err != nil {
-		LOG_ERROR("VSS_DELETE", "Error while unmounting snapshot")
+		LOG_WARN("VSS_DELETE", "Error while unmounting snapshot: %v", err)
 		return
 	}
 
 	err = exec.Command("tmutil", "deletelocalsnapshots", snapshotDate).Run()
 	if err != nil {
-		LOG_ERROR("VSS_DELETE", "Error while deleting local snapshot")
+		LOG_WARN("VSS_DELETE", "Error while deleting local snapshot: %v", err)
 		return
 	}
 
 	err = os.RemoveAll(snapshotPath)
 	if err != nil {
-		LOG_ERROR("VSS_DELETE", "Error while deleting temporary mount directory")
+		LOG_WARN("VSS_DELETE", "Error while deleting temporary mount directory: %v", err)
 		return
 	}
 
@@ -150,12 +149,13 @@ func CreateShadowCopy(top string, shadowCopy bool, timeoutInSeconds int) (shadow
 		return top
 	}
 
-	colonPos := strings.IndexByte(tmutilOutput, ':')
+	snapshotDateRegex := regexp.MustCompile(`:\s+([0-9\-]+)`)
-	if colonPos < 0 {
+	matched := snapshotDateRegex.FindStringSubmatch(tmutilOutput)
+	if matched == nil {
 		LOG_ERROR("VSS_CREATE", "Snapshot creation failed: %s", tmutilOutput)
 		return top
 	}
-	snapshotDate = strings.TrimSpace(tmutilOutput[colonPos+1:])
+	snapshotDate = matched[1]
 
 	tmutilOutput, err = CommandWithTimeout(timeoutInSeconds, "tmutil", "listlocalsnapshots", ".")
 	if err != nil {
@@ -164,17 +164,17 @@ func CreateShadowCopy(top string, shadowCopy bool, timeoutInSeconds int) (shadow
 	}
 	snapshotName := "com.apple.TimeMachine." + snapshotDate
  
-	r := regexp.MustCompile(`(?m)^(.+` + snapshotDate + `.*)$`)
+	snapshotNameRegex := regexp.MustCompile(`(?m)^(.+` + snapshotDate + `.*)$`)
-	snapshotNames := r.FindStringSubmatch(tmutilOutput)
+	matched = snapshotNameRegex.FindStringSubmatch(tmutilOutput)
-	if len(snapshotNames) > 0 {
+	if len(matched) > 0 {
-		snapshotName = snapshotNames[0]
+		snapshotName = matched[0]
 	} else {
-		LOG_WARN("VSS_CREATE", "Error while using 'tmutil listlocalsnapshots' to find snapshot name. Will fallback to 'com.apple.TimeMachine.SNAPSHOT_DATE'")
+		LOG_INFO("VSS_CREATE", "Can't find the snapshot name with 'tmutil listlocalsnapshots'; fallback to %s", snapshotName)
 	}
  
 	// Mount snapshot as readonly and hide from GUI i.e. Finder
 	_, err = CommandWithTimeout(timeoutInSeconds,
-		"/sbin/mount", "-t", "apfs", "-o", "nobrowse,-r,-s="+snapshotName, "/", snapshotPath)
+		"/sbin/mount", "-t", "apfs", "-o", "nobrowse,-r,-s="+snapshotName, "/System/Volumes/Data", snapshotPath)
 	if err != nil {
 		LOG_ERROR("VSS_CREATE", "Error while mounting snapshot: %v", err)
 		return top

src/duplicacy_snapshotmanager.go

@@ -1017,7 +1017,50 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 		manager.ShowStatistics(snapshotMap, chunkSizeMap, chunkUniqueMap, chunkSnapshotMap)
 	}
 
-	if checkChunks && !checkFiles {
+	// Don't verify chunks with -files
+	if !checkChunks || checkFiles {
+		return true
+	}
+
+	// This contains chunks that have been verifed in previous checks and is loaded from
+	// .duplicacy/cache/storage/verified_chunks.  Note that it contains the chunk ids not chunk
+	// hashes.
+	verifiedChunks := make(map[string]int64)
+	verifiedChunksFile := "verified_chunks"
+
+	manager.fileChunk.Reset(false)
+	err = manager.snapshotCache.DownloadFile(0, verifiedChunksFile, manager.fileChunk)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			LOG_WARN("SNAPSHOT_VERIFY", "Failed to load the file containing verified chunks: %v", err)
+		}
+	} else {
+		err = json.Unmarshal(manager.fileChunk.GetBytes(), &verifiedChunks)
+		if err != nil {
+			LOG_WARN("SNAPSHOT_VERIFY", "Failed to parse the file containing verified chunks: %v", err)
+		}
+	}
+	numberOfVerifiedChunks := len(verifiedChunks)
+
+	saveVerifiedChunks := func() {
+		if len(verifiedChunks) > numberOfVerifiedChunks {
+			var description []byte
+			description, err = json.Marshal(verifiedChunks)
+			if err != nil {
+				LOG_WARN("SNAPSHOT_VERIFY", "Failed to create a json file for the set of verified chunks: %v", err)
+			} else {
+				err = manager.snapshotCache.UploadFile(0, verifiedChunksFile, description)
+				if err != nil {
+					LOG_WARN("SNAPSHOT_VERIFY", "Failed to save the verified chunks file: %v", err)
+				} else {
+					LOG_INFO("SNAPSHOT_VERIFY", "Added %d chunks to the list of verified chunks", len(verifiedChunks) - numberOfVerifiedChunks)
+				}
+			}
+		}
+	}
+	defer saveVerifiedChunks()
+	RunAtError = saveVerifiedChunks
+
 	manager.chunkDownloader.snapshotCache = nil
 	LOG_INFO("SNAPSHOT_VERIFY", "Verifying %d chunks", len(*allChunkHashes))
 
@@ -1028,7 +1071,15 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 	// some metadata chunks so the index doesn't start with 0.
 	chunkIndex := -1
 
+	skippedChunks := 0
 	for chunkHash := range *allChunkHashes {
+		if len(verifiedChunks) > 0 {
+			chunkID := manager.config.GetChunkIDFromHash(chunkHash)
+			if _, found := verifiedChunks[chunkID]; found {
+				skippedChunks++
+				continue
+			}
+		}
 		chunkHashes = append(chunkHashes, chunkHash)
 		if chunkIndex == -1 {
 			chunkIndex = manager.chunkDownloader.AddChunk(chunkHash)
@@ -1037,13 +1088,19 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 		}
 	}
 
+	if skippedChunks > 0 {
+		LOG_INFO("SNAPSHOT_VERIFY", "Skipped %d chunks that have already been verified before", skippedChunks)
+	}
+
 	var downloadedChunkSize int64
-		totalChunks := len(*allChunkHashes)
+	totalChunks := len(chunkHashes)
 	for i := 0; i < totalChunks; i++ {
 		chunk := manager.chunkDownloader.WaitForChunk(i + chunkIndex)
+		chunkID := manager.config.GetChunkIDFromHash(chunkHashes[i])
 		if chunk.isBroken {
 			continue
 		}
+		verifiedChunks[chunkID] = startTime.Unix()
 		downloadedChunkSize += int64(chunk.GetLength())
 
 		elapsedTime := time.Now().Sub(startTime).Seconds()
@@ -1051,15 +1108,13 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 		remainingTime := int64(float64(totalChunks - i - 1) / float64(i + 1) * elapsedTime)
 		percentage := float64(i + 1) / float64(totalChunks) * 100.0
 		LOG_INFO("VERIFY_PROGRESS", "Verified chunk %s (%d/%d), %sB/s %s %.1f%%",
-					 manager.config.GetChunkIDFromHash(chunkHashes[i]), i + 1, totalChunks,
+					chunkID, i + 1, totalChunks, PrettySize(speed), PrettyTime(remainingTime), percentage)
-					 PrettySize(speed), PrettyTime(remainingTime), percentage)
 	}
 
 	if manager.chunkDownloader.NumberOfFailedChunks > 0 {
-			LOG_ERROR("SNAPSHOT_VERIFY", "%d out of %d chunks are corrupted", manager.chunkDownloader.NumberOfFailedChunks, totalChunks)
+		LOG_ERROR("SNAPSHOT_VERIFY", "%d out of %d chunks are corrupted", manager.chunkDownloader.NumberOfFailedChunks, len(*allChunkHashes))
 	} else {
-			LOG_INFO("SNAPSHOT_VERIFY", "All %d chunks have been successfully verified", totalChunks)
+		LOG_INFO("SNAPSHOT_VERIFY", "All %d chunks have been successfully verified", len(*allChunkHashes))
-		}
 	}
 	return true
 }

src/duplicacy_utils_others.go

@@ -13,7 +13,7 @@ import (
 	"path/filepath"
 	"syscall"
 
-	"github.com/gilbertchen/xattr"
+	"github.com/pkg/xattr"
 )
 
 func Readlink(path string) (isRegular bool, s string, err error) {
@@ -50,11 +50,11 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
 func (entry *Entry) ReadAttributes(top string) {
 
 	fullPath := filepath.Join(top, entry.Path)
-	attributes, _ := xattr.Listxattr(fullPath)
+	attributes, _ := xattr.List(fullPath)
 	if len(attributes) > 0 {
 		entry.Attributes = make(map[string][]byte)
 		for _, name := range attributes {
-			attribute, err := xattr.Getxattr(fullPath, name)
+			attribute, err := xattr.Get(fullPath, name)
 			if err == nil {
 				entry.Attributes[name] = attribute
 			}
@@ -63,24 +63,25 @@ func (entry *Entry) ReadAttributes(top string) {
 }
 
 func (entry *Entry) SetAttributesToFile(fullPath string) {
-	names, _ := xattr.Listxattr(fullPath)
+	names, _ := xattr.List(fullPath)
 
 	for _, name := range names {
 
+
 		newAttribute, found := entry.Attributes[name]
 		if found {
-			oldAttribute, _ := xattr.Getxattr(fullPath, name)
+			oldAttribute, _ := xattr.Get(fullPath, name)
 			if !bytes.Equal(oldAttribute, newAttribute) {
-				xattr.Setxattr(fullPath, name, newAttribute)
+				xattr.Set(fullPath, name, newAttribute)
 			}
 			delete(entry.Attributes, name)
 		} else {
-			xattr.Removexattr(fullPath, name)
+			xattr.Remove(fullPath, name)
 		}
 	}
 
 	for name, attribute := range entry.Attributes {
-		xattr.Setxattr(fullPath, name, attribute)
+		xattr.Set(fullPath, name, attribute)
 	}
 
 }