Bump version to 2.1.0

Implement OpenStack Swift backend

Gopkg.lock

@@ -7,11 +7,17 @@
   revision = "2d3a6656c17a60b0815b7e06ab0be04eacb6e613"
   version = "v0.16.0"
 
+[[projects]]
+  name = "github.com/Azure/azure-sdk-for-go"
+  packages = ["version"]
+  revision = "b7fadebe0e7f5c5720986080a01495bd8d27be37"
+  version = "v14.2.0"
+
 [[projects]]
   name = "github.com/Azure/go-autorest"
   packages = ["autorest","autorest/adal","autorest/azure","autorest/date"]
-  revision = "c67b24a8e30d876542a85022ebbdecf0e5a935e8"
-  version = "v9.4.1"
+  revision = "0ae36a9e544696de46fdadb7b0d5fb38af48c063"
+  version = "v10.2.0"
 
 [[projects]]
   branch = "master"
@@ -38,16 +44,16 @@
   version = "v3.1.0"
 
 [[projects]]
+  branch = "master"
   name = "github.com/gilbertchen/azure-sdk-for-go"
   packages = ["storage"]
-  revision = "2d49bb8f2cee530cc16f1f1a9f0aae763dee257d"
-  version = "v10.2.1-beta"
+  revision = "bbf89bd4d716c184f158d1e1428c2dbef4a18307"
 
 [[projects]]
+  branch = "master"
   name = "github.com/gilbertchen/cli"
   packages = ["."]
-  revision = "565493f259bf868adb54d45d5f4c68d405117adf"
-  version = "v1.2.0"
+  revision = "1de0a1836ce9c3ae1bf737a0869c4f04f28a7f98"
 
 [[projects]]
   branch = "master"
@@ -120,12 +126,24 @@
   packages = ["."]
   revision = "2788f0dbd16903de03cb8186e5c7d97b69ad387b"
 
+[[projects]]
+  name = "github.com/marstr/guid"
+  packages = ["."]
+  revision = "8bd9a64bf37eb297b492a4101fb28e80ac0b290f"
+  version = "v1.1.0"
+
 [[projects]]
   branch = "master"
   name = "github.com/minio/blake2b-simd"
   packages = ["."]
   revision = "3f5f724cb5b182a5c278d6d3d55b40e7f8c2efb4"
 
+[[projects]]
+  branch = "master"
+  name = "github.com/ncw/swift"
+  packages = ["."]
+  revision = "ae9f0ea1605b9aa6434ed5c731ca35d83ba67c55"
+
 [[projects]]
   name = "github.com/pkg/errors"
   packages = ["."]
@@ -139,10 +157,10 @@
   version = "1.0.0"
 
 [[projects]]
-  name = "github.com/satori/uuid"
+  name = "github.com/satori/go.uuid"
   packages = ["."]
-  revision = "879c5887cd475cd7864858769793b2ceb0d44feb"
-  version = "v1.1.0"
+  revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3"
+  version = "v1.2.0"
 
 [[projects]]
   branch = "master"
@@ -207,6 +225,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "95a162eedee5e915fbd1917c3ba5021e646aa2f13a542c7cbeb02bcf30a3acb9"
+  inputs-digest = "eff5ae2d9507f0d62cd2e5bdedebb5c59d64f70f476b087c01c35d4a5e1be72d"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -39,11 +39,11 @@
 
 [[constraint]]
   name = "github.com/gilbertchen/azure-sdk-for-go"
-  version = "10.2.1-beta"
+  branch = "master"
 
 [[constraint]]
+  branch = "master"
   name = "github.com/gilbertchen/cli"
-  version = "1.2.0"
 
 [[constraint]]
   branch = "master"

README.md

@@ -35,10 +35,10 @@ With Duplicacy, you can back up files to local or networked drives, SFTP servers
 
 | Type         |   Storage (monthly)    |   Upload           |    Download    |    API Charge   |
 |:------------:|:-------------:|:------------------:|:--------------:|:-----------:|
-| Amazon S3    | $0.023/GB | free | $0.09/GB | [yes](https://aws.amazon.com/s3/pricing/) |
-| Wasabi       | $3.99 first 1TB <br> $0.0039/GB additional | free | $.04/GB | no |
-| DigitalOcean Spaces| $5 first 250GB <br> $0.02/GB additional | free | first 1TB free <br> $0.01/GB additional| no |
-| Backblaze B2 | $0.005/GB | free | $0.02/GB | [yes](https://www.backblaze.com/b2/b2-transactions-price.html) |
+| Amazon S3    | $0.023/GB | free | $0.090/GB | [yes](https://aws.amazon.com/s3/pricing/) |
+| Wasabi       | $3.99 first 1TB <br> $0.0039/GB additional | free | $0.04/GB | no |
+| DigitalOcean Spaces| $5 first 250GB <br> $0.020/GB additional | free | first 1TB free <br> $0.01/GB additional| no |
+| Backblaze B2 | 10GB free <br> $0.005/GB | free | 1GB free/day <br> $0.02/GB | [yes](https://www.backblaze.com/b2/b2-transactions-price.html) |
 | Google Cloud Storage| $0.026/GB | free |$ 0.12/GB | [yes](https://cloud.google.com/storage/pricing) |
 | Google Drive | 15GB free <br> $1.99/100GB <br> $9.99/TB | free | free | no |
 | Microsoft Azure | $0.0184/GB | free | free | [yes](https://azure.microsoft.com/en-us/pricing/details/storage/blobs/) |

duplicacy/duplicacy_main.go

@@ -16,6 +16,9 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"net/http"
+
+	_ "net/http/pprof"
 
 	"github.com/gilbertchen/cli"
 
@@ -138,6 +141,15 @@ func setGlobalOptions(context *cli.Context) {
 		ScriptEnabled = false
 	}
 
+	address := context.GlobalString("profile")
+	if address != "" {
+		go func() {
+			http.ListenAndServe(address, nil)
+		}()
+	}
+
+
+
 	duplicacy.RunInBackground = context.GlobalBool("background")
 }
 
@@ -216,7 +228,10 @@ func configRepository(context *cli.Context, init bool) {
 	var storageURL string
 
 	if init {
-		storageName = "default"
+		storageName = context.String("storage-name")
+		if len(storageName) == 0 {
+			storageName = "default"
+		}
 		snapshotID = context.Args()[0]
 		storageURL = context.Args()[1]
 	} else {
@@ -589,11 +604,45 @@ func changePassword(context *cli.Context) {
 		iterations = duplicacy.CONFIG_DEFAULT_ITERATIONS
 	}
 
+	description, err := json.MarshalIndent(config, "", "    ")
+	if err != nil {
+		duplicacy.LOG_ERROR("CONFIG_MARSHAL", "Failed to marshal the config: %v", err)
+		return
+	}
+
+	configPath := path.Join(duplicacy.GetDuplicacyPreferencePath(), "config")
+	err = ioutil.WriteFile(configPath, description, 0600)
+	if err != nil {
+		duplicacy.LOG_ERROR("CONFIG_SAVE", "Failed to save the old config to %s: %v", configPath, err)
+		return
+	}
+	duplicacy.LOG_INFO("CONFIG_SAVE", "The old config has been temporarily saved to %s", configPath)
+
+	removeLocalCopy := false
+	defer func() {
+		if removeLocalCopy {
+			err = os.Remove(configPath)
+			if err != nil {
+				duplicacy.LOG_WARN("CONFIG_CLEAN", "Failed to delete %s: %v", configPath, err)
+			} else {
+				duplicacy.LOG_INFO("CONFIG_CLEAN", "The local copy of the old config has been removed")
+			}
+		}
+	} ()
+
+	err = storage.DeleteFile(0, "config")
+	if err != nil {
+		duplicacy.LOG_ERROR("CONFIG_DELETE", "Failed to delete the old config from the storage: %v", err)
+		return
+	}
+
 	duplicacy.UploadConfig(storage, config, newPassword, iterations)
 
 	duplicacy.SavePassword(*preference, "password", newPassword)
 
 	duplicacy.LOG_INFO("STORAGE_SET", "The password for storage %s has been changed", preference.StorageURL)
+
+	removeLocalCopy = true
 }
 
 func backupRepository(context *cli.Context) {
@@ -1147,6 +1196,11 @@ func infoStorage(context *cli.Context) {
 		DoNotSavePassword: true,
 	}
 
+	storageName := context.String("storage-name")
+	if storageName != "" {
+		preference.Name = storageName
+	}
+
 	if resetPasswords {
 		// We don't want password entered for the info command to overwrite the saved password for the default storage,
 		// so we simply assign an empty name.
@@ -1170,6 +1224,19 @@ func infoStorage(context *cli.Context) {
 	} else {
 		config.Print()
 	}
+
+	dirs, _, err := storage.ListFiles(0, "snapshots/")
+	if err != nil {
+		duplicacy.LOG_WARN("STORAGE_LIST", "Failed to list repository ids: %v", err)
+		return
+	}
+
+	for _, dir := range dirs {
+		if len(dir) > 0 && dir[len(dir)-1] == '/' {
+			duplicacy.LOG_INFO("STORAGE_SNAPSHOT", "%s", dir[0:len(dir) - 1])
+		}
+	}
+
 }
 
 func main() {
@@ -1204,7 +1271,7 @@ func main() {
 				},
 				cli.IntFlag{
 					Name:     "iterations",
-					Usage:    "the number of iterations used in storage key deriviation (default is 16384)",
+					Usage:    "the number of iterations used in storage key derivation (default is 16384)",
 					Argument: "<i>",
 				},
 				cli.StringFlag{
@@ -1212,6 +1279,11 @@ func main() {
 					Usage:    "alternate location for the .duplicacy directory (absolute or relative to current directory)",
 					Argument: "<path>",
 				},
+				cli.StringFlag{
+					Name:     "storage-name",
+					Usage:    "assign a name to the storage",
+					Argument: "<name>",
+				},
 			},
 			Usage:     "Initialize the storage if necessary and the current directory as the repository",
 			ArgsUsage: "<snapshot id> <storage url>",
@@ -1247,7 +1319,7 @@ func main() {
 				},
 				cli.BoolFlag{
 					Name:  "dry-run",
-					Usage: "Dry run for testing, don't backup anything. Use with -stats and -d",
+					Usage: "dry run for testing, don't backup anything. Use with -stats and -d",
 				},
 				cli.BoolFlag{
 					Name:  "vss",
@@ -1510,7 +1582,7 @@ func main() {
 				},
 				cli.StringSliceFlag{
 					Name:     "t",
-					Usage:    "delete snapshots with the specifed tags",
+					Usage:    "delete snapshots with the specified tags",
 					Argument: "<tag>",
 				},
 				cli.StringSliceFlag{
@@ -1524,7 +1596,7 @@ func main() {
 				},
 				cli.BoolFlag{
 					Name:  "exclusive",
-					Usage: "assume exclusive acess to the storage (disable two-step fossil collection)",
+					Usage: "assume exclusive access to the storage (disable two-step fossil collection)",
 				},
 				cli.BoolFlag{
 					Name:  "dry-run, d",
@@ -1564,7 +1636,7 @@ func main() {
 				},
 				cli.IntFlag{
 					Name:     "iterations",
-					Usage:    "the number of iterations used in storage key deriviation (default is 16384)",
+					Usage:    "the number of iterations used in storage key derivation (default is 16384)",
 					Argument: "<i>",
 				},
 			},
@@ -1598,7 +1670,7 @@ func main() {
 				},
 				cli.IntFlag{
 					Name:     "iterations",
-					Usage:    "the number of iterations used in storage key deriviation (default is 16384)",
+					Usage:    "the number of iterations used in storage key derivation (default is 16384)",
 					Argument: "<i>",
 				},
 				cli.StringFlag{
@@ -1720,6 +1792,11 @@ func main() {
 					Usage:    "retrieve saved passwords from the specified repository",
 					Argument: "<repository directory>",
 				},
+				cli.StringFlag{
+					Name:     "storage-name",
+					Usage:    "the storage name to be assigned to the storage url",
+					Argument: "<name>",
+				},
 				cli.BoolFlag{
 					Name:  "reset-passwords",
 					Usage: "take passwords from input rather than keychain/keyring",
@@ -1756,13 +1833,19 @@ func main() {
 			Name:  "background",
 			Usage: "read passwords, tokens, or keys only from keychain/keyring or env",
 		},
-	}
+		cli.StringFlag{
+			Name:     "profile",
+			Value:    "",
+			Usage:    "enable the profiling tool and listen on the specified address:port",
+			Argument: "<address:port>",
+		},
+}
 
 	app.HideVersion = true
 	app.Name = "duplicacy"
 	app.HelpName = "duplicacy"
 	app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
-	app.Version = "2.0.10"
+	app.Version = "2.1.0"
 
 	// If the program is interrupted, call the RunAtError function.
 	c := make(chan os.Signal, 1)

src/duplicacy_b2client.go

@@ -153,7 +153,7 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 			return response.Body, response.Header, response.ContentLength, nil
 		}
 
-		LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode)
+		LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s %s' returned status code %d", method, url, response.StatusCode)
 
 		io.Copy(ioutil.Discard, response.Body)
 		response.Body.Close()
@@ -170,7 +170,6 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 			continue
 		} else if response.StatusCode == 404 {
 			if http.MethodHead == method {
-				LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode)
 				return nil, nil, 0, nil
 			}
 		} else if response.StatusCode == 416 {
@@ -580,7 +579,7 @@ func (client *B2Client) UploadFile(filePath string, content []byte, rateLimit in
 		LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
 
 		if response.StatusCode == 401 {
-			LOG_INFO("BACKBLAZE_UPLOAD", "Re-authorizatoin required")
+			LOG_INFO("BACKBLAZE_UPLOAD", "Re-authorization required")
 			client.UploadURL = ""
 			client.UploadToken = ""
 			continue

src/duplicacy_b2storage.go

@@ -210,6 +210,7 @@ func (storage *B2Storage) GetFileInfo(threadIndex int, filePath string) (exist b
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *B2Storage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 
+	filePath = strings.Replace(filePath, " ", "%20", -1)
 	readCloser, _, err := storage.clients[threadIndex].DownloadFile(filePath)
 	if err != nil {
 		return err
@@ -223,6 +224,7 @@ func (storage *B2Storage) DownloadFile(threadIndex int, filePath string, chunk *
 
 // UploadFile writes 'content' to the file at 'filePath'.
 func (storage *B2Storage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
+	filePath = strings.Replace(filePath, " ", "%20", -1)
 	return storage.clients[threadIndex].UploadFile(filePath, content, storage.UploadRateLimit/len(storage.clients))
 }
 

src/duplicacy_backupmanager.go

@@ -242,6 +242,9 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 				}
 			}
 
+			LOG_DEBUG("CHUNK_INCOMPLETE", "The incomplete snapshot contains %d files and %d chunks", len(incompleteSnapshot.Files), len(incompleteSnapshot.ChunkHashes))
+			LOG_DEBUG("CHUNK_INCOMPLETE", "Last chunk in the incomplete snapshot that exist in the storage: %d", lastCompleteChunk)
+
 			// Only keep those files whose chunks exist in the cache
 			var files []*Entry
 			for _, file := range incompleteSnapshot.Files {
@@ -281,7 +284,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 	// we simply treat all files as if they were new, and break them into chunks.
 	// Otherwise, we need to find those that are new or recently modified
 
-	if remoteSnapshot.Revision == 0 && incompleteSnapshot == nil {
+	if (remoteSnapshot.Revision == 0 || !quickMode) && incompleteSnapshot == nil {
 		modifiedEntries = localSnapshot.Files
 		for _, entry := range modifiedEntries {
 			totalModifiedFileSize += entry.Size
@@ -747,7 +750,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 	}
 
 	remoteSnapshot := manager.SnapshotManager.DownloadSnapshot(manager.snapshotID, revision)
-	manager.SnapshotManager.DownloadSnapshotContents(remoteSnapshot, patterns)
+	manager.SnapshotManager.DownloadSnapshotContents(remoteSnapshot, patterns, true)
 
 	localSnapshot, _, _, err := CreateSnapshotFromDirectory(manager.snapshotID, top)
 	if err != nil {
@@ -915,9 +918,8 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 			totalFileSize, downloadedFileSize, startDownloadingTime) {
 			downloadedFileSize += file.Size
 			downloadedFiles = append(downloadedFiles, file)
-			file.RestoreMetadata(fullPath, nil, setOwner)
 		}
-
+		file.RestoreMetadata(fullPath, nil, setOwner)
 	}
 
 	if deleteMode && len(patterns) == 0 {

src/duplicacy_backupmanager_test.go

@@ -227,11 +227,11 @@ func TestBackupManager(t *testing.T) {
 
 	time.Sleep(time.Duration(delay) * time.Second)
 	if testFixedChunkSize {
-		if !ConfigStorage(storage, 16384, 100, 64*1024, 64*1024, 64*1024, password, nil) {
+		if !ConfigStorage(storage, 16384, 100, 64*1024, 64*1024, 64*1024, password, nil, false) {
 			t.Errorf("Failed to initialize the storage")
 		}
 	} else {
-		if !ConfigStorage(storage, 16384, 100, 64*1024, 256*1024, 16*1024, password, nil) {
+		if !ConfigStorage(storage, 16384, 100, 64*1024, 256*1024, 16*1024, password, nil, false) {
 			t.Errorf("Failed to initialize the storage")
 		}
 	}

src/duplicacy_chunkdownloader.go

@@ -298,38 +298,57 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
 	// will be set up before the encryption
 	chunk.Reset(false)
 
-	// Find the chunk by ID first.
-	chunkPath, exist, _, err := downloader.storage.FindChunk(threadIndex, chunkID, false)
-	if err != nil {
-		LOG_ERROR("DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
-		return false
-	}
+	const MaxDownloadAttempts = 3
+	for downloadAttempt := 0; ; downloadAttempt++ {
 
-	if !exist {
-		// No chunk is found.  Have to find it in the fossil pool again.
-		chunkPath, exist, _, err = downloader.storage.FindChunk(threadIndex, chunkID, true)
+		// Find the chunk by ID first.
+		chunkPath, exist, _, err := downloader.storage.FindChunk(threadIndex, chunkID, false)
 		if err != nil {
 			LOG_ERROR("DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
 			return false
 		}
 
 		if !exist {
-			// A chunk is not found.  This is a serious error and hopefully it will never happen.
+			// No chunk is found.  Have to find it in the fossil pool again.
+			fossilPath, exist, _, err := downloader.storage.FindChunk(threadIndex, chunkID, true)
 			if err != nil {
-				LOG_FATAL("DOWNLOAD_CHUNK", "Chunk %s can't be found: %v", chunkID, err)
-			} else {
-				LOG_FATAL("DOWNLOAD_CHUNK", "Chunk %s can't be found", chunkID)
+				LOG_ERROR("DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
+				return false
 			}
-			return false
-		}
-		LOG_DEBUG("CHUNK_FOSSIL", "Chunk %s has been marked as a fossil", chunkID)
-	}
 
-	const MaxDownloadAttempts = 3
-	for downloadAttempt := 0; ; downloadAttempt++ {
+			if !exist {
+				// Retry for the Hubic backend as it may return 404 even when the chunk exists
+				if _, ok := downloader.storage.(*HubicStorage); ok && downloadAttempt < MaxDownloadAttempts {
+					LOG_WARN("DOWNLOAD_RETRY", "Failed to find the chunk %s; retrying", chunkID)
+					continue
+				}
+
+				// A chunk is not found.  This is a serious error and hopefully it will never happen.
+				if err != nil {
+					LOG_FATAL("DOWNLOAD_CHUNK", "Chunk %s can't be found: %v", chunkID, err)
+				} else {
+					LOG_FATAL("DOWNLOAD_CHUNK", "Chunk %s can't be found", chunkID)
+				}
+				return false
+			}
+
+			// We can't download the fossil directly.  We have to turn it back into a regular chunk and try
+			// downloading again.
+			err = downloader.storage.MoveFile(threadIndex, fossilPath, chunkPath)
+			if err != nil {
+				LOG_FATAL("DOWNLOAD_CHUNK", "Failed to resurrect chunk %s: %v", chunkID, err)
+				return false
+			}
+
+			LOG_WARN("DOWNLOAD_RESURRECT", "Fossil %s has been resurrected", chunkID)
+			continue
+		}
+
 		err = downloader.storage.DownloadFile(threadIndex, chunkPath, chunk)
 		if err != nil {
-			if err == io.ErrUnexpectedEOF && downloadAttempt < MaxDownloadAttempts {
+			_, isHubic := downloader.storage.(*HubicStorage)
+			// Retry on EOF or if it is a Hubic backend as it may return 404 even when the chunk exists
+			if (err == io.ErrUnexpectedEOF || isHubic) && downloadAttempt < MaxDownloadAttempts {
 				LOG_WARN("DOWNLOAD_RETRY", "Failed to download the chunk %s: %v; retrying", chunkID, err)
 				chunk.Reset(false)
 				continue
@@ -368,7 +387,7 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
 
 	if len(cachedPath) > 0 {
 		// Save a copy to the local snapshot cache
-		err = downloader.snapshotCache.UploadFile(threadIndex, cachedPath, chunk.GetBytes())
+		err := downloader.snapshotCache.UploadFile(threadIndex, cachedPath, chunk.GetBytes())
 		if err != nil {
 			LOG_WARN("DOWNLOAD_CACHE", "Failed to add the chunk %s to the snapshot cache: %v", chunkID, err)
 		}

src/duplicacy_gcdstorage.go

@@ -24,11 +24,16 @@ import (
 	"google.golang.org/api/googleapi"
 )
 
+var (
+	GCDFileMimeType = "application/octet-stream"
+	GCDDirectoryMimeType = "application/vnd.google-apps.folder"
+)
+
 type GCDStorage struct {
 	StorageBase
 
 	service     *drive.Service
-	idCache     map[string]string
+	idCache     map[string]string  // only directories are saved in this cache
 	idCacheLock sync.Mutex
 	backoffs    []int // desired backoff time in seconds for each thread
 	attempts    []int // number of failed attempts since last success for each thread
@@ -165,7 +170,7 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles
 
 	startToken := ""
 
-	query := "'" + parentID + "' in parents "
+	query := "'" + parentID + "' in parents and trashed = false "
 	if listFiles && !listDirectories {
 		query += "and mimeType != 'application/vnd.google-apps.folder'"
 	} else if !listFiles && !listDirectories {
@@ -209,7 +214,7 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str
 	var err error
 
 	for {
-		query := "name = '" + name + "' and '" + parentID + "' in parents"
+		query := "name = '" + name + "' and '" + parentID + "' in parents and trashed = false "
 		fileList, err = storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Do()
 
 		if retry, e := storage.shouldRetry(threadIndex, err); e == nil && !retry {
@@ -227,7 +232,7 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str
 
 	file := fileList.Files[0]
 
-	return file.Id, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil
+	return file.Id, file.MimeType == GCDDirectoryMimeType, file.Size, nil
 }
 
 // getIDFromPath returns the id of the given path.  If 'createDirectories' is true, create the given path and all its
@@ -283,10 +288,10 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat
 			}
 			fileID = currentID
 			continue
-		} else {
+		} else if isDir {
 			storage.savePathID(current, fileID)
 		}
-		if i != len(names)-1 && !isDir {
+		if i != len(names) - 1 && !isDir {
 			return "", fmt.Errorf("Path '%s' is not a directory", current)
 		}
 	}
@@ -332,11 +337,13 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
 		storage.attempts[i] = 0
 	}
 
-	storagePathID, err := storage.getIDFromPath(0, storagePath, false)
+	storagePathID, err := storage.getIDFromPath(0, storagePath, true)
 	if err != nil {
 		return nil, err
 	}
 
+	// Reset the id cache and start with 'storagePathID' as the root
+	storage.idCache = make(map[string]string)
 	storage.idCache[""] = storagePathID
 
 	for _, dir := range []string{"chunks", "snapshots", "fossils"} {
@@ -379,8 +386,8 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 		subDirs := []string{}
 
 		for _, file := range files {
-			storage.savePathID("snapshots/"+file.Name, file.Id)
-			subDirs = append(subDirs, file.Name+"/")
+			storage.savePathID("snapshots/" + file.Name, file.Id)
+			subDirs = append(subDirs, file.Name + "/")
 		}
 		return subDirs, nil, nil
 	} else if strings.HasPrefix(dir, "snapshots/") {
@@ -400,7 +407,6 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 		files := []string{}
 
 		for _, entry := range entries {
-			storage.savePathID(dir+"/"+entry.Name, entry.Id)
 			files = append(files, entry.Name)
 		}
 		return files, nil, nil
@@ -420,7 +426,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 				return nil, nil, err
 			}
 			for _, entry := range entries {
-				if entry.MimeType != "application/vnd.google-apps.folder" {
+				if entry.MimeType != GCDDirectoryMimeType {
 					name := entry.Name
 					if strings.HasPrefix(parent, "fossils") {
 						name = parent + "/" + name + ".fsl"
@@ -432,9 +438,9 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 					files = append(files, name)
 					sizes = append(sizes, entry.Size)
 				} else {
-					parents = append(parents, parent+"/"+entry.Name)
+					parents = append(parents, parent+ "/" + entry.Name)
+					storage.savePathID(parent + "/" + entry.Name, entry.Id)
 				}
-				storage.savePathID(parent+"/"+entry.Name, entry.Id)
 			}
 		}
 		return files, sizes, nil
@@ -474,9 +480,12 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
 	from = storage.convertFilePath(from)
 	to = storage.convertFilePath(to)
 
-	fileID, ok := storage.findPathID(from)
-	if !ok {
-		return fmt.Errorf("Attempting to rename file %s with unknown id", from)
+	fileID, err := storage.getIDFromPath(threadIndex, from, false)
+	if err != nil {
+		return fmt.Errorf("Failed to retrieve the id of '%s': %v", from, err)
+	}
+	if fileID == "" {
+		return fmt.Errorf("The file '%s' to be moved does not exist", from)
 	}
 
 	fromParent := path.Dir(from)
@@ -505,8 +514,6 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
 		}
 	}
 
-	storage.savePathID(to, storage.getPathID(from))
-	storage.deletePathID(from)
 	return nil
 }
 
@@ -539,21 +546,22 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err
 	}
 	name := path.Base(dir)
 
-	file := &drive.File{
-		Name:     name,
-		MimeType: "application/vnd.google-apps.folder",
-		Parents:  []string{parentID},
-	}
+	var file *drive.File
 
 	for {
+		file = &drive.File{
+			Name:     name,
+			MimeType: GCDDirectoryMimeType,
+			Parents:  []string{parentID},
+		}
+
 		file, err = storage.service.Files.Create(file).Fields("id").Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			break
 		} else {
 
 			// Check if the directory has already been created by other thread
-			exist, _, _, newErr := storage.GetFileInfo(threadIndex, dir)
-			if newErr == nil && exist {
+			if _, ok := storage.findPathID(dir); ok {
 				return nil
 			}
 
@@ -577,36 +585,29 @@ func (storage *GCDStorage) GetFileInfo(threadIndex int, filePath string) (exist
 	filePath = storage.convertFilePath(filePath)
 
 	fileID, ok := storage.findPathID(filePath)
-	if !ok {
-		dir := path.Dir(filePath)
-		if dir == "." {
-			dir = ""
-		}
-		dirID, err := storage.getIDFromPath(threadIndex, dir, false)
-		if err != nil {
-			return false, false, 0, err
-		}
-		if dirID == "" {
-			return false, false, 0, nil
-		}
-
-		fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath))
-		if fileID != "" {
-			storage.savePathID(filePath, fileID)
-		}
-		return fileID != "", isDir, size, err
+	if ok {
+		// Only directories are saved in the case so this must be a directory
+		return true, true, 0, nil
 	}
 
-	for {
-		file, err := storage.service.Files.Get(fileID).Fields("id, mimeType").Do()
-		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
-			return true, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil
-		} else if retry {
-			continue
-		} else {
-			return false, false, 0, err
-		}
+	dir := path.Dir(filePath)
+	if dir == "." {
+		dir = ""
 	}
+	dirID, err := storage.getIDFromPath(threadIndex, dir, false)
+	if err != nil {
+		return false, false, 0, err
+	}
+	if dirID == "" {
+		return false, false, 0, nil
+	}
+
+	fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath))
+	if fileID != "" && isDir {
+		storage.savePathID(filePath, fileID)
+	}
+	return fileID != "", isDir, size, err
+
 }
 
 // DownloadFile reads the file at 'filePath' into the chunk.
@@ -656,7 +657,7 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content
 
 	file := &drive.File{
 		Name:     path.Base(filePath),
-		MimeType: "application/octet-stream",
+		MimeType: GCDFileMimeType,
 		Parents:  []string{parentID},
 	}
 

src/duplicacy_hubicclient.go

@@ -72,7 +72,7 @@ func NewHubicClient(tokenFile string) (*HubicClient, error) {
 					KeepAlive: 30 * time.Second,
 				}).Dial,
 				TLSHandshakeTimeout:   60 * time.Second,
-				ResponseHeaderTimeout: 30 * time.Second,
+				ResponseHeaderTimeout: 300 * time.Second,
 				ExpectContinueTimeout: 10 * time.Second,
 			},
 		},
@@ -82,7 +82,7 @@ func NewHubicClient(tokenFile string) (*HubicClient, error) {
 		CredentialLock: &sync.Mutex{},
 	}
 
-	err = client.RefreshToken()
+	err = client.RefreshToken(false)
 	if err != nil {
 		return nil, err
 	}
@@ -100,7 +100,7 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
 	var response *http.Response
 
 	backoff := 1
-	for i := 0; i < 8; i++ {
+	for i := 0; i < 11; i++ {
 
 		LOG_DEBUG("HUBIC_CALL", "%s %s", method, url)
 
@@ -151,6 +151,13 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
 
 		response, err = client.HTTPClient.Do(request)
 		if err != nil {
+			if url != HubicCredentialURL {
+				retryAfter := time.Duration((0.5 + rand.Float32()) * 1000.0 * float32(backoff))
+				LOG_INFO("HUBIC_CALL", "%s %s returned an error: %v; retry after %d milliseconds", method, url, err, retryAfter)
+				time.Sleep(retryAfter * time.Millisecond)
+				backoff *= 2
+				continue
+			}
 			return nil, 0, "", err
 		}
 
@@ -179,7 +186,7 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
 				return nil, 0, "", HubicError{Status: response.StatusCode, Message: "Authorization error when retrieving credentials"}
 			}
 
-			err = client.RefreshToken()
+			err = client.RefreshToken(true)
 			if err != nil {
 				return nil, 0, "", err
 			}
@@ -190,7 +197,13 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
 			}
 			continue
 		} else if response.StatusCode >= 500 && response.StatusCode < 600 {
-			retryAfter := time.Duration(rand.Float32() * 1000.0 * float32(backoff))
+			retryAfter := time.Duration((0.5 + rand.Float32()) * 1000.0 * float32(backoff))
+			LOG_INFO("HUBIC_RETRY", "Response status: %d; retry after %d milliseconds", response.StatusCode, retryAfter)
+			time.Sleep(retryAfter * time.Millisecond)
+			backoff *= 2
+			continue
+		} else if response.StatusCode == 408 {
+			retryAfter := time.Duration((0.5 + rand.Float32()) * 1000.0 * float32(backoff))
 			LOG_INFO("HUBIC_RETRY", "Response status: %d; retry after %d milliseconds", response.StatusCode, retryAfter)
 			time.Sleep(retryAfter * time.Millisecond)
 			backoff *= 2
@@ -203,11 +216,11 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
 	return nil, 0, "", fmt.Errorf("Maximum number of retries reached")
 }
 
-func (client *HubicClient) RefreshToken() (err error) {
+func (client *HubicClient) RefreshToken(force bool) (err error) {
 	client.TokenLock.Lock()
 	defer client.TokenLock.Unlock()
 
-	if client.Token.Valid() {
+	if !force && client.Token.Valid() {
 		return nil
 	}
 

src/duplicacy_hubicstorage.go

@@ -106,17 +106,19 @@ func (storage *HubicStorage) ListFiles(threadIndex int, dir string) ([]string, [
 	} else {
 		files := []string{}
 		sizes := []int64{}
-		entries, err := storage.client.ListEntries(storage.storageDir + "/chunks")
+		entries, err := storage.client.ListEntries(storage.storageDir + "/" + dir)
 		if err != nil {
 			return nil, nil, err
 		}
 
 		for _, entry := range entries {
 			if entry.Type == "application/directory" {
-				continue
+				files = append(files,  entry.Name + "/")
+				sizes = append(sizes, 0)
+			} else {
+				files = append(files, entry.Name)
+				sizes = append(sizes, entry.Size)
 			}
-			files = append(files, entry.Name)
-			sizes = append(sizes, entry.Size)
 		}
 		return files, sizes, nil
 	}

src/duplicacy_oneclient.go

@@ -65,6 +65,8 @@ func NewOneDriveClient(tokenFile string) (*OneDriveClient, error) {
 		TokenLock:  &sync.Mutex{},
 	}
 
+	client.RefreshToken(false)
+
 	return client, nil
 }
 
@@ -154,7 +156,7 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 				return nil, 0, OneDriveError{Status: response.StatusCode, Message: "Authorization error when refreshing token"}
 			}
 
-			err = client.RefreshToken()
+			err = client.RefreshToken(true)
 			if err != nil {
 				return nil, 0, err
 			}
@@ -178,11 +180,11 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 	return nil, 0, fmt.Errorf("Maximum number of retries reached")
 }
 
-func (client *OneDriveClient) RefreshToken() (err error) {
+func (client *OneDriveClient) RefreshToken(force bool) (err error) {
 	client.TokenLock.Lock()
 	defer client.TokenLock.Unlock()
 
-	if client.Token.Valid() {
+	if !force && client.Token.Valid() {
 		return nil
 	}
 

src/duplicacy_sftpstorage.go

@@ -53,7 +53,7 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
 	}
 
 	if server == "sftp.hidrive.strato.com" {
-		sftpConfig.Ciphers = []string{"aes128-cbc", "aes128-ctr", "aes256-ctr"}
+		sftpConfig.Ciphers = []string{"aes128-ctr", "aes256-ctr"}
 	}
 
 	serverAddress := fmt.Sprintf("%s:%d", server, port)

src/duplicacy_snapshotmanager.go

@@ -269,7 +269,7 @@ func (manager *SnapshotManager) DownloadSequence(sequence []string) (content []b
 	return content
 }
 
-func (manager *SnapshotManager) DownloadSnapshotFileSequence(snapshot *Snapshot, patterns []string) bool {
+func (manager *SnapshotManager) DownloadSnapshotFileSequence(snapshot *Snapshot, patterns []string, attributesNeeded bool) bool {
 
 	manager.CreateChunkDownloader()
 
@@ -304,7 +304,8 @@ func (manager *SnapshotManager) DownloadSnapshotFileSequence(snapshot *Snapshot,
 			return false
 		}
 
-		if len(patterns) != 0 && !MatchPath(entry.Path, patterns) {
+		// If we don't need the attributes or the file isn't included we clear the attributes to save memory
+		if !attributesNeeded || (len(patterns) != 0 && !MatchPath(entry.Path, patterns)) {
 			entry.Attributes = nil
 		}
 
@@ -347,9 +348,9 @@ func (manager *SnapshotManager) DownloadSnapshotSequence(snapshot *Snapshot, seq
 // DownloadSnapshotContents loads all chunk sequences in a snapshot.  A snapshot, when just created, only contains
 // some metadata and theree sequence representing files, chunk hashes, and chunk lengths.  This function must be called
 // for the actual content of the snapshot to be usable.
-func (manager *SnapshotManager) DownloadSnapshotContents(snapshot *Snapshot, patterns []string) bool {
+func (manager *SnapshotManager) DownloadSnapshotContents(snapshot *Snapshot, patterns []string, attributesNeeded bool) bool {
 
-	manager.DownloadSnapshotFileSequence(snapshot, patterns)
+	manager.DownloadSnapshotFileSequence(snapshot, patterns, attributesNeeded)
 	manager.DownloadSnapshotSequence(snapshot, "chunks")
 	manager.DownloadSnapshotSequence(snapshot, "lengths")
 
@@ -553,7 +554,7 @@ func (manager *SnapshotManager) downloadLatestSnapshot(snapshotID string) (remot
 	}
 
 	if remote != nil {
-		manager.DownloadSnapshotContents(remote, nil)
+		manager.DownloadSnapshotContents(remote, nil, false)
 	}
 
 	return remote
@@ -679,7 +680,7 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
 			}
 
 			if showFiles {
-				manager.DownloadSnapshotFileSequence(snapshot, nil)
+				manager.DownloadSnapshotFileSequence(snapshot, nil, false)
 			}
 
 			if showFiles {
@@ -799,7 +800,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 			}
 
 			if checkFiles {
-				manager.DownloadSnapshotContents(snapshot, nil)
+				manager.DownloadSnapshotContents(snapshot, nil, false)
 				manager.VerifySnapshot(snapshot)
 				continue
 			}
@@ -817,7 +818,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 				if !found {
 					if !searchFossils {
 						missingChunks += 1
-						LOG_WARN("SNAPHOST_VALIDATE",
+						LOG_WARN("SNAPSHOT_VALIDATE",
 							"Chunk %s referenced by snapshot %s at revision %d does not exist",
 							chunkID, snapshotID, revision)
 						continue
@@ -825,14 +826,14 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 
 					chunkPath, exist, size, err := manager.storage.FindChunk(0, chunkID, true)
 					if err != nil {
-						LOG_ERROR("SNAPHOST_VALIDATE", "Failed to check the existence of chunk %s: %v",
+						LOG_ERROR("SNAPSHOT_VALIDATE", "Failed to check the existence of chunk %s: %v",
 							chunkID, err)
 						return false
 					}
 
 					if !exist {
 						missingChunks += 1
-						LOG_WARN("SNAPHOST_VALIDATE",
+						LOG_WARN("SNAPSHOT_VALIDATE",
 							"Chunk %s referenced by snapshot %s at revision %d does not exist",
 							chunkID, snapshotID, revision)
 						continue
@@ -841,7 +842,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 					if resurrect {
 						manager.resurrectChunk(chunkPath, chunkID)
 					} else {
-						LOG_WARN("SNAPHOST_FOSSIL", "Chunk %s referenced by snapshot %s at revision %d "+
+						LOG_WARN("SNAPSHOT_FOSSIL", "Chunk %s referenced by snapshot %s at revision %d "+
 							"has been marked as a fossil", chunkID, snapshotID, revision)
 					}
 
@@ -1113,6 +1114,14 @@ func (manager *SnapshotManager) RetrieveFile(snapshot *Snapshot, file *Entry, ou
 
 	manager.CreateChunkDownloader()
 
+	// Temporarily disable the snapshot cache of the download so that downloaded file chunks won't be saved
+	// to the cache.
+	snapshotCache := manager.chunkDownloader.snapshotCache
+	manager.chunkDownloader.snapshotCache = nil
+	defer func() {
+		manager.chunkDownloader.snapshotCache = snapshotCache
+	}()
+
 	fileHasher := manager.config.NewFileHasher()
 	alternateHash := false
 	if strings.HasPrefix(file.Hash, "#") {
@@ -1200,7 +1209,8 @@ func (manager *SnapshotManager) PrintFile(snapshotID string, revision int, path
 		patterns = []string{path}
 	}
 
-	if !manager.DownloadSnapshotContents(snapshot, patterns) {
+	// If no path is specified, we're printing the snapshot so we need all attributes
+	if !manager.DownloadSnapshotContents(snapshot, patterns, path == "") {
 		return false
 	}
 
@@ -1260,9 +1270,9 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []
 
 	if len(filePath) > 0 {
 
-		manager.DownloadSnapshotContents(leftSnapshot, nil)
+		manager.DownloadSnapshotContents(leftSnapshot, nil, false)
 		if rightSnapshot != nil && rightSnapshot.Revision != 0 {
-			manager.DownloadSnapshotContents(rightSnapshot, nil)
+			manager.DownloadSnapshotContents(rightSnapshot, nil, false)
 		}
 
 		var leftFile []byte
@@ -1338,9 +1348,9 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []
 	}
 
 	// We only need to decode the 'files' sequence, not 'chunkhashes' or 'chunklengthes'
-	manager.DownloadSnapshotFileSequence(leftSnapshot, nil)
+	manager.DownloadSnapshotFileSequence(leftSnapshot, nil, false)
 	if rightSnapshot != nil && rightSnapshot.Revision != 0 {
-		manager.DownloadSnapshotFileSequence(rightSnapshot, nil)
+		manager.DownloadSnapshotFileSequence(rightSnapshot, nil, false)
 	}
 
 	maxSize := int64(9)
@@ -1444,7 +1454,7 @@ func (manager *SnapshotManager) ShowHistory(top string, snapshotID string, revis
 	sort.Ints(revisions)
 	for _, revision := range revisions {
 		snapshot := manager.DownloadSnapshot(snapshotID, revision)
-		manager.DownloadSnapshotFileSequence(snapshot, nil)
+		manager.DownloadSnapshotFileSequence(snapshot, nil, false)
 		file := manager.FindFile(snapshot, filePath, true)
 
 		if file != nil {
@@ -1855,7 +1865,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 				}
 
 				if len(tagMap) > 0 {
-					if _, found := tagMap[snapshot.Tag]; found {
+					if _, found := tagMap[snapshot.Tag]; !found {
 						continue
 					}
 				}
@@ -2284,6 +2294,10 @@ func (manager *SnapshotManager) DownloadFile(path string, derivationKey string)
 		return nil
 	}
 
+	if len(derivationKey) > 64 {
+		derivationKey = derivationKey[len(derivationKey) - 64:]
+	}
+
 	err = manager.fileChunk.Decrypt(manager.config.FileKey, derivationKey)
 	if err != nil {
 		LOG_ERROR("DOWNLOAD_DECRYPT", "Failed to decrypt the file %s: %v", path, err)
@@ -2314,6 +2328,10 @@ func (manager *SnapshotManager) UploadFile(path string, derivationKey string, co
 		}
 	}
 
+	if len(derivationKey) > 64 {
+		derivationKey = derivationKey[len(derivationKey) - 64:]
+	}
+
 	err := manager.fileChunk.Encrypt(manager.config.FileKey, derivationKey)
 	if err != nil {
 		LOG_ERROR("UPLOAD_File", "Failed to encrypt the file %s: %v", path, err)

src/duplicacy_snapshotmanager_test.go

@@ -107,6 +107,9 @@ func createTestSnapshotManager(testDir string) *SnapshotManager {
 	snapshotCache.CreateDirectory(0, "snapshots")
 
 	snapshotManager.snapshotCache = snapshotCache
+
+	SetDuplicacyPreferencePath(testDir + "/.duplicacy")
+
 	return snapshotManager
 }
 
@@ -140,7 +143,7 @@ func uploadRandomChunk(manager *SnapshotManager, chunkSize int) string {
 	return uploadTestChunk(manager, content)
 }
 
-func createTestSnapshot(manager *SnapshotManager, snapshotID string, revision int, startTime int64, endTime int64, chunkHashes []string) {
+func createTestSnapshot(manager *SnapshotManager, snapshotID string, revision int, startTime int64, endTime int64, chunkHashes []string, tag string) {
 
 	snapshot := &Snapshot{
 		ID:          snapshotID,
@@ -148,6 +151,7 @@ func createTestSnapshot(manager *SnapshotManager, snapshotID string, revision in
 		StartTime:   startTime,
 		EndTime:     endTime,
 		ChunkHashes: chunkHashes,
+		Tag:         tag,
 	}
 
 	var chunkHashesInHex []string
@@ -239,12 +243,12 @@ func TestSingleRepositoryPrune(t *testing.T) {
 	now := time.Now().Unix()
 	day := int64(24 * 3600)
 	t.Logf("Creating 1 snapshot")
-	createTestSnapshot(snapshotManager, "repository1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
+	createTestSnapshot(snapshotManager, "repository1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
 	checkTestSnapshots(snapshotManager, 1, 2)
 
 	t.Logf("Creating 2 snapshots")
-	createTestSnapshot(snapshotManager, "repository1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
-	createTestSnapshot(snapshotManager, "repository1", 3, now-1*day-3600, now-1*day-60, []string{chunkHash3, chunkHash4})
+	createTestSnapshot(snapshotManager, "repository1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
+	createTestSnapshot(snapshotManager, "repository1", 3, now-1*day-3600, now-1*day-60, []string{chunkHash3, chunkHash4}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 0)
 
 	t.Logf("Removing snapshot repository1 revision 1 with --exclusive")
@@ -257,7 +261,7 @@ func TestSingleRepositoryPrune(t *testing.T) {
 
 	t.Logf("Creating 1 snapshot")
 	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
-	createTestSnapshot(snapshotManager, "repository1", 4, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5})
+	createTestSnapshot(snapshotManager, "repository1", 4, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5}, "tag")
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Prune without removing any snapshots -- fossils will be deleted")
@@ -282,9 +286,9 @@ func TestSingleHostPrune(t *testing.T) {
 	now := time.Now().Unix()
 	day := int64(24 * 3600)
 	t.Logf("Creating 3 snapshots")
-	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
-	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
-	createTestSnapshot(snapshotManager, "vm2@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash3, chunkHash4})
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
+	createTestSnapshot(snapshotManager, "vm2@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash3, chunkHash4}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 0)
 
 	t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
@@ -297,7 +301,7 @@ func TestSingleHostPrune(t *testing.T) {
 
 	t.Logf("Creating 1 snapshot")
 	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
-	createTestSnapshot(snapshotManager, "vm2@host1", 2, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5})
+	createTestSnapshot(snapshotManager, "vm2@host1", 2, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 2)
 
 	t.Logf("Prune without removing any snapshots -- fossils will be deleted")
@@ -323,9 +327,9 @@ func TestMultipleHostPrune(t *testing.T) {
 	now := time.Now().Unix()
 	day := int64(24 * 3600)
 	t.Logf("Creating 3 snapshot")
-	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
-	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
-	createTestSnapshot(snapshotManager, "vm2@host2", 1, now-3*day-3600, now-3*day-60, []string{chunkHash3, chunkHash4})
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
+	createTestSnapshot(snapshotManager, "vm2@host2", 1, now-3*day-3600, now-3*day-60, []string{chunkHash3, chunkHash4}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 0)
 
 	t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
@@ -338,7 +342,7 @@ func TestMultipleHostPrune(t *testing.T) {
 
 	t.Logf("Creating 1 snapshot")
 	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
-	createTestSnapshot(snapshotManager, "vm2@host2", 2, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5})
+	createTestSnapshot(snapshotManager, "vm2@host2", 2, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 2)
 
 	t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
@@ -347,7 +351,7 @@ func TestMultipleHostPrune(t *testing.T) {
 
 	t.Logf("Creating 1 snapshot")
 	chunkHash6 := uploadRandomChunk(snapshotManager, chunkSize)
-	createTestSnapshot(snapshotManager, "vm1@host1", 3, now+1*day-3600, now+1*day, []string{chunkHash5, chunkHash6})
+	createTestSnapshot(snapshotManager, "vm1@host1", 3, now+1*day-3600, now+1*day, []string{chunkHash5, chunkHash6}, "tag")
 	checkTestSnapshots(snapshotManager, 4, 2)
 
 	t.Logf("Prune without removing any snapshots -- fossils will be deleted")
@@ -371,8 +375,8 @@ func TestPruneAndResurrect(t *testing.T) {
 	now := time.Now().Unix()
 	day := int64(24 * 3600)
 	t.Logf("Creating 2 snapshots")
-	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
-	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
 	checkTestSnapshots(snapshotManager, 2, 0)
 
 	t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
@@ -381,7 +385,7 @@ func TestPruneAndResurrect(t *testing.T) {
 
 	t.Logf("Creating 1 snapshot")
 	chunkHash4 := uploadRandomChunk(snapshotManager, chunkSize)
-	createTestSnapshot(snapshotManager, "vm1@host1", 4, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash1})
+	createTestSnapshot(snapshotManager, "vm1@host1", 4, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash1}, "tag")
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Prune without removing any snapshots -- one fossil will be resurrected")
@@ -406,10 +410,10 @@ func TestInactiveHostPrune(t *testing.T) {
 	now := time.Now().Unix()
 	day := int64(24 * 3600)
 	t.Logf("Creating 3 snapshot")
-	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
-	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
 	// Host2 is inactive
-	createTestSnapshot(snapshotManager, "vm2@host2", 1, now-7*day-3600, now-7*day-60, []string{chunkHash3, chunkHash4})
+	createTestSnapshot(snapshotManager, "vm2@host2", 1, now-7*day-3600, now-7*day-60, []string{chunkHash3, chunkHash4}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 0)
 
 	t.Logf("Removing snapshot vm1@host1 revision 1")
@@ -422,7 +426,7 @@ func TestInactiveHostPrune(t *testing.T) {
 
 	t.Logf("Creating 1 snapshot")
 	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
-	createTestSnapshot(snapshotManager, "vm1@host1", 3, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5})
+	createTestSnapshot(snapshotManager, "vm1@host1", 3, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 2)
 
 	t.Logf("Prune without removing any snapshots -- fossils will be deleted")
@@ -448,7 +452,7 @@ func TestRetentionPolicy(t *testing.T) {
 	day := int64(24 * 3600)
 	t.Logf("Creating 30 snapshots")
 	for i := 0; i < 30; i++ {
-		createTestSnapshot(snapshotManager, "vm1@host1", i+1, now-int64(30-i)*day-3600, now-int64(30-i)*day-60, []string{chunkHashes[i]})
+		createTestSnapshot(snapshotManager, "vm1@host1", i+1, now-int64(30-i)*day-3600, now-int64(30-i)*day-60, []string{chunkHashes[i]}, "tag")
 	}
 
 	checkTestSnapshots(snapshotManager, 30, 0)
@@ -465,3 +469,35 @@ func TestRetentionPolicy(t *testing.T) {
 	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"3:14", "2:7"}, false, true, []string{}, false, false, false)
 	checkTestSnapshots(snapshotManager, 12, 0)
 }
+
+func TestRetentionPolicyAndTag(t *testing.T) {
+
+	setTestingT(t)
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test", "snapshot_test")
+
+	snapshotManager := createTestSnapshotManager(testDir)
+
+	chunkSize := 1024
+	var chunkHashes []string
+	for i := 0; i < 30; i++ {
+		chunkHashes = append(chunkHashes, uploadRandomChunk(snapshotManager, chunkSize))
+	}
+
+	now := time.Now().Unix()
+	day := int64(24 * 3600)
+	t.Logf("Creating 30 snapshots")
+	for i := 0; i < 30; i++ {
+		tag := "auto"
+		if i % 3 == 0 {
+			tag = "manual"
+		}
+		createTestSnapshot(snapshotManager, "vm1@host1", i+1, now-int64(30-i)*day-3600, now-int64(30-i)*day-60, []string{chunkHashes[i]}, tag)
+	}
+
+	checkTestSnapshots(snapshotManager, 30, 0)
+
+	t.Logf("Removing snapshot vm1@host1 0:20 with --exclusive and --tag manual")
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{"manual"}, []string{"0:7"}, false, true, []string{}, false, false, false)
+	checkTestSnapshots(snapshotManager, 22, 0)
+}

src/duplicacy_storage.go

@@ -109,7 +109,7 @@ func (storage *StorageBase) SetNestingLevels(config *Config) {
 		exist, _, _, err := storage.DerivedStorage.GetFileInfo(0, "nesting")
 		if err == nil && exist {
 			nestingFile := CreateChunk(CreateConfig(), true)
-			if storage.DerivedStorage.DownloadFile(0, "config", nestingFile) == nil {
+			if storage.DerivedStorage.DownloadFile(0, "nesting", nestingFile) == nil {
 				var nesting struct {
 					ReadLevels []int `json:"read-levels"`
 					WriteLevel int   `json:"write-level"`
@@ -560,6 +560,16 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 		}
 		SavePassword(preference, "hubic_token", tokenFile)
 		return hubicStorage
+	} else if matched[1] == "swift" {
+		prompt := fmt.Sprintf("Enter the OpenStack Swift key:")
+		key := GetPassword(preference, "swift_key", prompt, true, resetPassword)
+		swiftStorage, err := CreateSwiftStorage(storageURL[8:], key, threads)
+		if err != nil {
+			LOG_ERROR("STORAGE_CREATE", "Failed to load the OpenStack Swift storage at %s: %v", storageURL, err)
+			return nil
+		}
+		SavePassword(preference, "swift_key", key)
+		return swiftStorage
 	} else {
 		LOG_ERROR("STORAGE_CREATE", "The storage type '%s' is not supported", matched[1])
 		return nil

src/duplicacy_storage_test.go

@@ -138,6 +138,10 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
 		storage, err := CreateHubicStorage(config["token_file"], config["storage_path"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
+	} else if testStorageName == "memset" {
+		storage, err := CreateSwiftStorage(config["storage_url"], config["key"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else {
 		return nil, fmt.Errorf("Invalid storage named: %s", testStorageName)
 	}

src/duplicacy_swiftstorage.go

@@ -0,0 +1,251 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ncw/swift"
+)
+
+type SwiftStorage struct {
+	StorageBase
+
+	connection *swift.Connection
+	container  string
+	storageDir string
+	threads    int
+}
+
+// CreateSwiftStorage creates an OpenStack Swift storage object.  storageURL is in the form of 
+// `user@authURL/container/path?arg1=value1&arg2=value2``
+func CreateSwiftStorage(storageURL string, key string, threads int) (storage *SwiftStorage, err error) {
+
+	// This is the map to store all arguments
+	arguments := make(map[string]string)
+
+	// Check if there are arguments provided as a query string
+	if strings.Contains(storageURL, "?") {
+		urlAndArguments := strings.SplitN(storageURL, "?", 2)
+		storageURL = urlAndArguments[0]
+		for _, pair := range strings.Split(urlAndArguments[1], "&") {
+			if strings.Contains(pair, "=") {
+				keyAndValue := strings.Split(pair, "=")
+				arguments[keyAndValue[0]] = keyAndValue[1]
+			}
+		}
+	}
+
+	// Take out the user name if there is one
+	if strings.Contains(storageURL, "@") {
+		userAndURL := strings.Split(storageURL, "@")
+		arguments["user"] = userAndURL[0]
+		storageURL = userAndURL[1]
+	}
+
+	// The version is used to split authURL and container/path
+	versions := []string{"/v1/", "/v1.0/", "/v2/", "/v2.0/", "/v3/", "/v3.0/", "/v4/", "/v4.0/"}
+	storageDir := ""
+	for _, version := range versions {
+		if strings.Contains(storageURL, version) {
+			urlAndStorageDir := strings.SplitN(storageURL, version, 2)
+			storageURL = urlAndStorageDir[0] + version[0:len(version)-1]
+			storageDir = urlAndStorageDir[1]
+		}
+	}
+
+	// If no container/path is specified, find them from the arguments
+	if storageDir == "" {
+		storageDir = arguments["storage_dir"]
+	}
+
+	// Now separate the container name from the storage path
+	container := ""
+	if strings.Contains(storageDir, "/") {
+		containerAndStorageDir := strings.SplitN(storageDir, "/", 2)
+		container = containerAndStorageDir[0]
+		storageDir = containerAndStorageDir[1]
+		if len(storageDir) > 0 && storageDir[len(storageDir)-1] != '/' {
+			storageDir += "/"
+		}
+	} else {
+		container = storageDir
+		storageDir = ""
+	}
+
+	// Number of retries on err
+	retries := 4
+	if value, ok := arguments["retries"]; ok {
+		retries, _ = strconv.Atoi(value)
+	}
+
+	// Connect channel timeout
+	connectionTimeout := 10
+	if value, ok := arguments["connection_timeout"]; ok {
+		connectionTimeout, _ = strconv.Atoi(value)
+	}
+
+	// Data channel timeout
+	timeout := 60
+	if value, ok := arguments["timeout"]; ok {
+		timeout, _ = strconv.Atoi(value)
+	}
+
+	// Auth version; default to auto-detect
+	authVersion := 0
+	if value, ok := arguments["auth_version"]; ok {
+		authVersion, _ = strconv.Atoi(value)
+	}
+
+	// Allow http to be used by setting "protocol=http" in arguments
+	if _, ok := arguments["protocol"]; !ok {
+		arguments["protocol"] = "https"
+	}
+
+	// Please refer to https://godoc.org/github.com/ncw/swift#Connection
+	connection := swift.Connection{
+		Domain:         arguments["domain"],
+		DomainId:       arguments["domain_id"],
+		UserName:       arguments["user"],
+		UserId:         arguments["user_id"],
+		ApiKey:         key,
+		AuthUrl:        arguments["protocol"] + "://" + storageURL,
+		Retries:        retries,
+		UserAgent:      arguments["user_agent"],
+		ConnectTimeout: time.Duration(connectionTimeout) * time.Second,
+		Timeout:        time.Duration(timeout) * time.Second,
+		Region:         arguments["region"],
+		AuthVersion:    authVersion,
+		Internal:       false,
+		Tenant:         arguments["tenant"],
+		TenantId:       arguments["tenant_id"],
+		EndpointType:   swift.EndpointType(arguments["endpiont_type"]),
+		TenantDomain:   arguments["tenant_domain"],
+		TenantDomainId: arguments["tenant_domain_id"],
+		TrustId:        arguments["trust_id"],
+	}
+
+	_, _, err = connection.Container(container)
+	if err != nil {
+		return nil, err
+	}
+
+	storage = &SwiftStorage{
+		connection: &connection,
+		container:  container,
+		storageDir: storageDir,
+		threads:    threads,
+	}
+
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{1}, 1)
+	return storage, nil
+}
+
+// ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
+func (storage *SwiftStorage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) {
+	if len(dir) > 0 && dir[len(dir)-1] != '/' {
+		dir += "/"
+	}
+	isSnapshotDir := dir == "snapshots/"
+	dir = storage.storageDir + dir
+
+	options := swift.ObjectsOpts{
+		Prefix: dir,
+		Limit:  1000,
+	}
+
+	if isSnapshotDir {
+		options.Delimiter = '/'
+	}
+
+	objects, err := storage.connection.ObjectsAll(storage.container, &options)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for _, obj := range objects {
+		if isSnapshotDir {
+			if obj.SubDir != "" {
+				files = append(files, obj.SubDir[len(dir):])
+				sizes = append(sizes, 0)
+			}
+		} else {
+			files = append(files, obj.Name[len(dir):])
+			sizes = append(sizes, obj.Bytes)
+		}
+	}
+
+	return files, sizes, nil
+}
+
+// DeleteFile deletes the file or directory at 'filePath'.
+func (storage *SwiftStorage) DeleteFile(threadIndex int, filePath string) (err error) {
+	return storage.connection.ObjectDelete(storage.container, storage.storageDir+filePath)
+}
+
+// MoveFile renames the file.
+func (storage *SwiftStorage) MoveFile(threadIndex int, from string, to string) (err error) {
+	return storage.connection.ObjectMove(storage.container, storage.storageDir+from,
+		storage.container, storage.storageDir+to)
+}
+
+// CreateDirectory creates a new directory.
+func (storage *SwiftStorage) CreateDirectory(threadIndex int, dir string) (err error) {
+	// Does nothing as directories do not exist in OpenStack Swift
+	return nil
+}
+
+// GetFileInfo returns the information about the file or directory at 'filePath'.
+func (storage *SwiftStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
+	object, _, err := storage.connection.Object(storage.container, storage.storageDir+filePath)
+
+	if err != nil {
+		if err == swift.ObjectNotFound {
+			return false, false, 0, nil
+		} else {
+			return false, false, 0, err
+		}
+	}
+
+	return true, false, object.Bytes, nil
+}
+
+// DownloadFile reads the file at 'filePath' into the chunk.
+func (storage *SwiftStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
+
+	file, _, err := storage.connection.ObjectOpen(storage.container, storage.storageDir+filePath, false, nil)
+	if err != nil {
+		return err
+	}
+	_, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit/storage.threads)
+	return err
+}
+
+// UploadFile writes 'content' to the file at 'filePath'.
+func (storage *SwiftStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
+	reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.threads)
+	_, err = storage.connection.ObjectPut(storage.container, storage.storageDir+filePath, reader, true, "", "application/duplicacy", nil)
+	return err
+}
+
+// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
+// managing snapshots.
+func (storage *SwiftStorage) IsCacheNeeded() bool { return true }
+
+// If the 'MoveFile' method is implemented.
+func (storage *SwiftStorage) IsMoveFileImplemented() bool { return true }
+
+// If the storage can guarantee strong consistency.
+func (storage *SwiftStorage) IsStrongConsistent() bool { return false }
+
+// If the storage supports fast listing of files names.
+func (storage *SwiftStorage) IsFastListing() bool { return true }
+
+// Enable the test mode.
+func (storage *SwiftStorage) EnableTestMode() {
+}

src/duplicacy_utils_others.go

@@ -69,7 +69,7 @@ func (entry *Entry) SetAttributesToFile(fullPath string) {
 		newAttribute, found := entry.Attributes[name]
 		if found {
 			oldAttribute, _ := xattr.Getxattr(fullPath, name)
-			if bytes.Equal(oldAttribute, newAttribute) {
+			if !bytes.Equal(oldAttribute, newAttribute) {
 				xattr.Setxattr(fullPath, name, newAttribute)
 			}
 			delete(entry.Attributes, name)