Add a storage prefix flat:// that can handle a flat chunk directory

src/duplicacy_backupmanager.go

@@ -76,7 +76,7 @@ func (manager *BackupManager) SetupSnapshotCache(storageName string) bool {
     preferencePath := GetDuplicacyPreferencePath()
     cacheDir := path.Join(preferencePath, "cache", storageName)
 
-    storage, err := CreateFileStorage(cacheDir, 1)
+    storage, err := CreateFileStorage(cacheDir, 2, 1)
     if err != nil {
         LOG_ERROR("BACKUP_CACHE", "Failed to create the snapshot cache dir: %v", err)
         return false

src/duplicacy_backupmanager_test.go

@@ -173,6 +173,9 @@ func TestBackupManager(t *testing.T) {
 
     os.Mkdir(testDir + "/repository1", 0700)
     os.Mkdir(testDir + "/repository1/dir1", 0700)
+    os.Mkdir(testDir + "/repository1/.duplicacy", 0700)
+    os.Mkdir(testDir + "/repository2", 0700)
+    os.Mkdir(testDir + "/repository2/.duplicacy", 0700)
 
     maxFileSize := 1000000
     //maxFileSize := 200000
@@ -215,14 +218,14 @@ func TestBackupManager(t *testing.T) {
 
     time.Sleep(time.Duration(delay) * time.Second)
 
-    SetDuplicacyPreferencePath(testDir + "/repository1")
+    SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
     backupManager := CreateBackupManager("host1", storage, testDir, password)
     backupManager.SetupSnapshotCache("default")
 
-    SetDuplicacyPreferencePath(testDir + "/repository1")
+    SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
     backupManager.Backup(testDir + "/repository1", /*quickMode=*/true, threads, "first", false, false)
     time.Sleep(time.Duration(delay) * time.Second)
-    SetDuplicacyPreferencePath(testDir + "/repository2")
+    SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
     backupManager.Restore(testDir + "/repository2", threads, /*inPlace=*/false, /*quickMode=*/false, threads, /*overwrite=*/true,
                           /*deleteMode=*/false, /*showStatistics=*/false, /*patterns=*/nil)
 
@@ -243,10 +246,10 @@ func TestBackupManager(t *testing.T) {
     modifyFile(testDir + "/repository1/file2", 0.2)
     modifyFile(testDir + "/repository1/dir1/file3", 0.3)
 
-    SetDuplicacyPreferencePath(testDir + "/repository1")
+    SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
     backupManager.Backup(testDir + "/repository1", /*quickMode=*/true, threads, "second", false, false)
     time.Sleep(time.Duration(delay) * time.Second)
-    SetDuplicacyPreferencePath(testDir + "/repository2")
+    SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
     backupManager.Restore(testDir + "/repository2", 2, /*inPlace=*/true, /*quickMode=*/true, threads, /*overwrite=*/true,
                           /*deleteMode=*/false, /*showStatistics=*/false, /*patterns=*/nil)
 
@@ -259,10 +262,10 @@ func TestBackupManager(t *testing.T) {
     }
 
     truncateFile(testDir + "/repository1/file2")
-    SetDuplicacyPreferencePath(testDir + "/repository1")
+    SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
     backupManager.Backup(testDir + "/repository1", /*quickMode=*/false, threads, "third", false, false)
     time.Sleep(time.Duration(delay) * time.Second)
-    SetDuplicacyPreferencePath(testDir + "/repository2")
+    SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
     backupManager.Restore(testDir + "/repository2", 3, /*inPlace=*/true, /*quickMode=*/false, threads, /*overwrite=*/true,
                           /*deleteMode=*/false, /*showStatistics=*/false, /*patterns=*/nil)
 
@@ -277,6 +280,7 @@ func TestBackupManager(t *testing.T) {
     // Remove file2 and dir1/file3 and restore them from revision 3
     os.Remove(testDir + "/repository1/file2")
     os.Remove(testDir + "/repository1/dir1/file3")
+    SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
     backupManager.Restore(testDir + "/repository1", 3, /*inPlace=*/true, /*quickMode=*/false, threads, /*overwrite=*/true,
                           /*deleteMode=*/false, /*showStatistics=*/false, /*patterns=*/[]string{"+file2", "+dir1/file3", "-*"})
 

src/duplicacy_filestorage.go

@@ -18,12 +18,13 @@ import (
 type FileStorage struct {
     RateLimitedStorage
 
+    minimumLevel int       // The minimum level of directories to dive into before searching for the chunk file.
     storageDir string
     numberOfThreads int
 }
 
 // CreateFileStorage creates a file storage.
-func CreateFileStorage(storageDir string, threads int) (storage *FileStorage, err error) {
+func CreateFileStorage(storageDir string, minimumLevel int, threads int) (storage *FileStorage, err error) {
 
     var stat os.FileInfo
 
@@ -45,6 +46,7 @@ func CreateFileStorage(storageDir string, threads int) (storage *FileStorage, er
 
     storage = &FileStorage {
         storageDir : storageDir,
+        minimumLevel: minimumLevel,
         numberOfThreads: threads,
     }
 
@@ -128,11 +130,8 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil
         suffix = ".fsl"
     }
 
-    // The minimum level of directories to dive into before searching for the chunk file.
-    minimumLevel := 2
-
     for level := 0; level * 2 < len(chunkID); level ++ {
-        if level >= minimumLevel {
+        if level >= storage.minimumLevel {
             filePath = path.Join(dir, chunkID[2 * level:]) + suffix
             if stat, err := os.Stat(filePath); err == nil && !stat.IsDir() {
                 return filePath[len(storage.storageDir) + 1:], true, stat.Size(), nil
@@ -149,7 +148,7 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil
             continue
         }
 
-        if level < minimumLevel {
+        if level < storage.minimumLevel {
             // Create the subdirectory if it doesn't exist.
 
             if err == nil && !stat.IsDir() {

src/duplicacy_snapshotmanager_test.go

@@ -95,14 +95,14 @@ func createTestSnapshotManager(testDir string) *SnapshotManager {
     os.RemoveAll(testDir)
     os.MkdirAll(testDir, 0700)
 
-    storage, _ := CreateFileStorage(testDir, 1)
+    storage, _ := CreateFileStorage(testDir, 2, 1)
     storage.CreateDirectory(0, "chunks")
     storage.CreateDirectory(0, "snapshots")
     config := CreateConfig()
     snapshotManager := CreateSnapshotManager(config, storage)
 
     cacheDir := path.Join(testDir, "cache")
-    snapshotCache, _ := CreateFileStorage(cacheDir, 1)
+    snapshotCache, _ := CreateFileStorage(cacheDir, 2, 1)
     snapshotCache.CreateDirectory(0, "chunks")
     snapshotCache.CreateDirectory(0, "snapshots")
 

src/duplicacy_storage.go

@@ -144,7 +144,16 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
     }
 
     if isFileStorage {
-        fileStorage, err := CreateFileStorage(storageURL, threads)
+        fileStorage, err := CreateFileStorage(storageURL, 2, threads)
+        if err != nil {
+            LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err)
+            return nil
+        }
+        return fileStorage
+    }
+
+    if strings.HasPrefix(storageURL, "flat://") {
+        fileStorage, err := CreateFileStorage(storageURL, 0, threads)
         if err != nil {
             LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err)
             return nil

src/duplicacy_storage_test.go

@@ -41,7 +41,7 @@ func init() {
 func loadStorage(localStoragePath string, threads int) (Storage, error) {
 
     if testStorageName == "" || testStorageName == "file" {
-        return CreateFileStorage(localStoragePath, threads)
+        return CreateFileStorage(localStoragePath, 2, threads)
     }
 
     config, err := ioutil.ReadFile("test_storage.conf")
@@ -61,7 +61,9 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
         return nil, fmt.Errorf("No storage named '%s' found", testStorageName)
     }
 
-    if testStorageName == "sftp" {
+    if testStorageName == "flat" {
+        return CreateFileStorage(localStoragePath, 0, threads)
+    } else if testStorageName == "sftp" {
         port, _ := strconv.Atoi(storage["port"])
         return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], storage["password"], threads)
     } else if testStorageName == "s3" {