Refactor the BackupManager options interface some more

- Move the pref based (repository associated) options back
to init of the backup manager. This is still a bit of an unholy mess but
the existing setup is a little insane.
There are mostly runtime associated settings, like revision, tag, thread
count, quick mode, show stats. Then there are the more fuzzy ones,
like allow failures, patterns. However something like set-owner, mostly
likely that's going to be static for a "repository", same with
exclude by attribute. In any event things that make sense to be
associated with the repository preferences will stay in a backup mananger
initialization time options structure, and no more growing of the
arguments list - go doesn't look good there.

RestoreOptions for the more ephemeral run associated settings.
The Backup routine with it's large argument list stays for now, but
will get revamped if anything new gets added.
This commit is contained in:
2023-10-06 17:13:51 -05:00
parent 52e8a8280e
commit f74ce50068
4 changed files with 112 additions and 109 deletions

View File

@@ -22,9 +22,7 @@ import (
"github.com/gilbertchen/cli" "github.com/gilbertchen/cli"
"io/ioutil" duplicacy "github.com/gilbertchen/duplicacy/src"
"github.com/gilbertchen/duplicacy/src"
) )
const ( const (
@@ -316,7 +314,7 @@ func configRepository(context *cli.Context, init bool) {
// write real path into .duplicacy file inside repository // write real path into .duplicacy file inside repository
duplicacyFileName := path.Join(repository, duplicacy.DUPLICACY_FILE) duplicacyFileName := path.Join(repository, duplicacy.DUPLICACY_FILE)
d1 := []byte(preferencePath) d1 := []byte(preferencePath)
err = ioutil.WriteFile(duplicacyFileName, d1, 0644) err = os.WriteFile(duplicacyFileName, d1, 0644)
if err != nil { if err != nil {
duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to write %s file inside repository %v", duplicacyFileName, err) duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to write %s file inside repository %v", duplicacyFileName, err)
return return
@@ -705,7 +703,7 @@ func changePassword(context *cli.Context) {
} }
configPath := path.Join(duplicacy.GetDuplicacyPreferencePath(), "config") configPath := path.Join(duplicacy.GetDuplicacyPreferencePath(), "config")
err = ioutil.WriteFile(configPath, description, 0600) err = os.WriteFile(configPath, description, 0600)
if err != nil { if err != nil {
duplicacy.LOG_ERROR("CONFIG_SAVE", "Failed to save the old config to %s: %v", configPath, err) duplicacy.LOG_ERROR("CONFIG_SAVE", "Failed to save the old config to %s: %v", configPath, err)
return return
@@ -789,7 +787,12 @@ func backupRepository(context *cli.Context) {
uploadRateLimit := context.Int("limit-rate") uploadRateLimit := context.Int("limit-rate")
enumOnly := context.Bool("enum-only") enumOnly := context.Bool("enum-only")
storage.SetRateLimits(0, uploadRateLimit) storage.SetRateLimits(0, uploadRateLimit)
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile, preference.ExcludeByAttribute) backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password,
&duplicacy.BackupManagerOptions{
NoBackupFile: preference.NobackupFile,
FiltersFile: preference.FiltersFile,
ExcludeByAttribute: preference.ExcludeByAttribute,
})
duplicacy.SavePassword(*preference, "password", password) duplicacy.SavePassword(*preference, "password", password)
backupManager.SetupSnapshotCache(preference.Name) backupManager.SetupSnapshotCache(preference.Name)
@@ -850,20 +853,6 @@ func restoreRepository(context *cli.Context) {
password = duplicacy.GetPassword(*preference, "password", "Enter storage password:", false, false) password = duplicacy.GetPassword(*preference, "password", "Enter storage password:", false, false)
} }
options := duplicacy.RestoreOptions{
InPlace: true,
QuickMode: !context.Bool("hash"),
Overwrite: context.Bool("overwrite"),
DeleteMode: context.Bool("delete"),
SetOwner: !context.Bool("ignore-owner"),
ShowStatistics: context.Bool("stats"),
AllowFailures: context.Bool("persist"),
ExcludeXattrs: preference.ExcludeXattrs,
NormalizeXattrs: preference.NormalizeXattrs,
IncludeSpecials: preference.IncludeSpecials,
FileFlagsMask: uint32(preference.FileFlagsMask),
}
var patterns []string var patterns []string
for _, pattern := range context.Args() { for _, pattern := range context.Args() {
@@ -880,20 +869,45 @@ func restoreRepository(context *cli.Context) {
patterns = append(patterns, pattern) patterns = append(patterns, pattern)
} }
options.Patterns = duplicacy.ProcessFilterLines(patterns, make([]string, 0)) patterns = duplicacy.ProcessFilterLines(patterns, make([]string, 0))
duplicacy.LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(duplicacy.RegexMap)) duplicacy.LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(duplicacy.RegexMap))
duplicacy.LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) duplicacy.LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns))
storage.SetRateLimits(context.Int("limit-rate"), 0) storage.SetRateLimits(context.Int("limit-rate"), 0)
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile, preference.ExcludeByAttribute)
excludeOwner := preference.ExcludeOwner
// TODO: for backward compat, eventually make them all overridable?
if context.IsSet("ignore-owner") {
excludeOwner = context.Bool("ignore-owner")
}
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password,
&duplicacy.BackupManagerOptions{
NoBackupFile: preference.NobackupFile,
FiltersFile: preference.FiltersFile,
ExcludeByAttribute: preference.ExcludeByAttribute,
SetOwner: excludeOwner,
ExcludeXattrs: preference.ExcludeXattrs,
NormalizeXattrs: preference.NormalizeXattrs,
IncludeSpecials: preference.IncludeSpecials,
FileFlagsMask: uint32(preference.FileFlagsMask),
})
duplicacy.SavePassword(*preference, "password", password) duplicacy.SavePassword(*preference, "password", password)
loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false) loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false)
backupManager.SetupSnapshotCache(preference.Name) backupManager.SetupSnapshotCache(preference.Name)
failed := backupManager.Restore(repository, revision, options) failed := backupManager.Restore(repository, revision, &duplicacy.RestoreOptions{
InPlace: true,
QuickMode: !context.Bool("hash"),
Overwrite: context.Bool("overwrite"),
DeleteMode: context.Bool("delete"),
ShowStatistics: context.Bool("stats"),
AllowFailures: context.Bool("persist"),
})
if failed > 0 { if failed > 0 {
duplicacy.LOG_ERROR("RESTORE_FAIL", "%d file(s) were not restored correctly", failed) duplicacy.LOG_ERROR("RESTORE_FAIL", "%d file(s) were not restored correctly", failed)
return return
@@ -933,7 +947,8 @@ func listSnapshots(context *cli.Context) {
tag := context.String("t") tag := context.String("t")
revisions := getRevisions(context) revisions := getRevisions(context)
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", preference.ExcludeByAttribute) backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password,
&duplicacy.BackupManagerOptions{ExcludeByAttribute: preference.ExcludeByAttribute})
duplicacy.SavePassword(*preference, "password", password) duplicacy.SavePassword(*preference, "password", password)
id := preference.SnapshotID id := preference.SnapshotID
@@ -989,7 +1004,7 @@ func checkSnapshots(context *cli.Context) {
tag := context.String("t") tag := context.String("t")
revisions := getRevisions(context) revisions := getRevisions(context)
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false) backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil)
duplicacy.SavePassword(*preference, "password", password) duplicacy.SavePassword(*preference, "password", password)
loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false) loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false)
@@ -1049,8 +1064,7 @@ func printFile(context *cli.Context) {
snapshotID = context.String("id") snapshotID = context.String("id")
} }
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil)
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false)
duplicacy.SavePassword(*preference, "password", password) duplicacy.SavePassword(*preference, "password", password)
loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false) loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false)
@@ -1108,7 +1122,7 @@ func diff(context *cli.Context) {
} }
compareByHash := context.Bool("hash") compareByHash := context.Bool("hash")
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false) backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil)
duplicacy.SavePassword(*preference, "password", password) duplicacy.SavePassword(*preference, "password", password)
loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false) loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false)
@@ -1153,7 +1167,7 @@ func showHistory(context *cli.Context) {
revisions := getRevisions(context) revisions := getRevisions(context)
showLocalHash := context.Bool("hash") showLocalHash := context.Bool("hash")
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false) backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil)
duplicacy.SavePassword(*preference, "password", password) duplicacy.SavePassword(*preference, "password", password)
backupManager.SetupSnapshotCache(preference.Name) backupManager.SetupSnapshotCache(preference.Name)
@@ -1216,7 +1230,7 @@ func pruneSnapshots(context *cli.Context) {
os.Exit(ArgumentExitCode) os.Exit(ArgumentExitCode)
} }
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false) backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil)
duplicacy.SavePassword(*preference, "password", password) duplicacy.SavePassword(*preference, "password", password)
backupManager.SetupSnapshotCache(preference.Name) backupManager.SetupSnapshotCache(preference.Name)
@@ -1261,7 +1275,7 @@ func copySnapshots(context *cli.Context) {
sourcePassword = duplicacy.GetPassword(*source, "password", "Enter source storage password:", false, false) sourcePassword = duplicacy.GetPassword(*source, "password", "Enter source storage password:", false, false)
} }
sourceManager := duplicacy.CreateBackupManager(source.SnapshotID, sourceStorage, repository, sourcePassword, "", "", false) sourceManager := duplicacy.CreateBackupManager(source.SnapshotID, sourceStorage, repository, sourcePassword, nil)
sourceManager.SetupSnapshotCache(source.Name) sourceManager.SetupSnapshotCache(source.Name)
duplicacy.SavePassword(*source, "password", sourcePassword) duplicacy.SavePassword(*source, "password", sourcePassword)
@@ -1296,7 +1310,7 @@ func copySnapshots(context *cli.Context) {
destinationStorage.SetRateLimits(0, context.Int("upload-limit-rate")) destinationStorage.SetRateLimits(0, context.Int("upload-limit-rate"))
destinationManager := duplicacy.CreateBackupManager(destination.SnapshotID, destinationStorage, repository, destinationManager := duplicacy.CreateBackupManager(destination.SnapshotID, destinationStorage, repository,
destinationPassword, "", "", false) destinationPassword, nil)
duplicacy.SavePassword(*destination, "password", destinationPassword) duplicacy.SavePassword(*destination, "password", destinationPassword)
destinationManager.SetupSnapshotCache(destination.Name) destinationManager.SetupSnapshotCache(destination.Name)
@@ -1421,7 +1435,7 @@ func benchmark(context *cli.Context) {
if storage == nil { if storage == nil {
return return
} }
duplicacy.Benchmark(repository, storage, int64(fileSize) * 1024 * 1024, chunkSize * 1024 * 1024, chunkCount, uploadThreads, downloadThreads) duplicacy.Benchmark(repository, storage, int64(fileSize)*1024*1024, chunkSize*1024*1024, chunkCount, uploadThreads, downloadThreads)
} }
func main() { func main() {
@@ -1460,8 +1474,8 @@ func main() {
Argument: "<level>", Argument: "<level>",
}, },
cli.BoolFlag{ cli.BoolFlag{
Name: "zstd", Name: "zstd",
Usage: "short for -zstd default", Usage: "short for -zstd default",
}, },
cli.IntFlag{ cli.IntFlag{
Name: "iterations", Name: "iterations",
@@ -1536,8 +1550,8 @@ func main() {
Argument: "<level>", Argument: "<level>",
}, },
cli.BoolFlag{ cli.BoolFlag{
Name: "zstd", Name: "zstd",
Usage: "short for -zstd default", Usage: "short for -zstd default",
}, },
cli.BoolFlag{ cli.BoolFlag{
Name: "vss", Name: "vss",
@@ -1570,7 +1584,6 @@ func main() {
Usage: "the maximum number of entries kept in memory (defaults to 1M)", Usage: "the maximum number of entries kept in memory (defaults to 1M)",
Argument: "<number>", Argument: "<number>",
}, },
}, },
Usage: "Save a snapshot of the repository to the storage", Usage: "Save a snapshot of the repository to the storage",
ArgsUsage: " ", ArgsUsage: " ",
@@ -1630,7 +1643,7 @@ func main() {
cli.BoolFlag{ cli.BoolFlag{
Name: "persist", Name: "persist",
Usage: "continue processing despite chunk errors or existing files (without -overwrite), reporting any affected files", Usage: "continue processing despite chunk errors or existing files (without -overwrite), reporting any affected files",
}, },
cli.StringFlag{ cli.StringFlag{
Name: "key-passphrase", Name: "key-passphrase",
Usage: "the passphrase to decrypt the RSA private key", Usage: "the passphrase to decrypt the RSA private key",
@@ -1988,8 +2001,8 @@ func main() {
Argument: "<level>", Argument: "<level>",
}, },
cli.BoolFlag{ cli.BoolFlag{
Name: "zstd", Name: "zstd",
Usage: "short for -zstd default", Usage: "short for -zstd default",
}, },
cli.IntFlag{ cli.IntFlag{
Name: "iterations", Name: "iterations",
@@ -2254,8 +2267,8 @@ func main() {
Usage: "add a comment to identify the process", Usage: "add a comment to identify the process",
}, },
cli.StringSliceFlag{ cli.StringSliceFlag{
Name: "suppress, s", Name: "suppress, s",
Usage: "suppress logs with the specified id", Usage: "suppress logs with the specified id",
Argument: "<id>", Argument: "<id>",
}, },
cli.BoolFlag{ cli.BoolFlag{

View File

@@ -32,32 +32,33 @@ type BackupManager struct {
SnapshotManager *SnapshotManager // the snapshot manager SnapshotManager *SnapshotManager // the snapshot manager
snapshotCache *FileStorage // for copies of chunks needed by snapshots snapshotCache *FileStorage // for copies of chunks needed by snapshots
config *Config // contains a number of options config *Config // contains a number of options
options BackupManagerOptions
nobackupFile string // don't backup directory when this file name is found
filtersFile string // the path to the filters file
excludeByAttribute bool // don't backup file based on file attribute
cachePath string cachePath string
} }
type BackupOptions struct { type BackupManagerOptions struct {
NoBackupFile string // don't backup directory when this file name is found
FiltersFile string // the path to the filters file
ExcludeByAttribute bool // don't backup file based on file attribute
SetOwner bool
ExcludeXattrs bool
NormalizeXattrs bool
IncludeFileFlags bool
IncludeSpecials bool
FileFlagsMask uint32
} }
type RestoreOptions struct { type RestoreOptions struct {
Threads int Threads int
Patterns []string Patterns []string
InPlace bool InPlace bool
QuickMode bool QuickMode bool
Overwrite bool Overwrite bool
DeleteMode bool DeleteMode bool
SetOwner bool ShowStatistics bool
ShowStatistics bool AllowFailures bool
AllowFailures bool
ExcludeXattrs bool
NormalizeXattrs bool
IncludeSpecials bool
FileFlagsMask uint32
} }
func (manager *BackupManager) SetDryRun(dryRun bool) { func (manager *BackupManager) SetDryRun(dryRun bool) {
@@ -71,7 +72,8 @@ func (manager *BackupManager) SetCompressionLevel(level int) {
// CreateBackupManager creates a backup manager using the specified 'storage'. 'snapshotID' is a unique id to // CreateBackupManager creates a backup manager using the specified 'storage'. 'snapshotID' is a unique id to
// identify snapshots created for this repository. 'top' is the top directory of the repository. 'password' is the // identify snapshots created for this repository. 'top' is the top directory of the repository. 'password' is the
// master key which can be nil if encryption is not enabled. // master key which can be nil if encryption is not enabled.
func CreateBackupManager(snapshotID string, storage Storage, top string, password string, nobackupFile string, filtersFile string, excludeByAttribute bool) *BackupManager { func CreateBackupManager(snapshotID string, storage Storage, top string, password string,
options *BackupManagerOptions) *BackupManager {
config, _, err := DownloadConfig(storage, password) config, _, err := DownloadConfig(storage, password)
if err != nil { if err != nil {
@@ -91,13 +93,11 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor
SnapshotManager: snapshotManager, SnapshotManager: snapshotManager,
config: config, config: config,
options: *options,
nobackupFile: nobackupFile, }
if options != nil {
filtersFile: filtersFile, backupManager.options = *options
excludeByAttribute: excludeByAttribute,
} }
if IsDebugging() { if IsDebugging() {
@@ -170,7 +170,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
LOG_INFO("BACKUP_KEY", "RSA encryption is enabled") LOG_INFO("BACKUP_KEY", "RSA encryption is enabled")
} }
if manager.excludeByAttribute { if manager.options.ExcludeByAttribute {
LOG_INFO("BACKUP_EXCLUDE", "Exclude files with no-backup attributes") LOG_INFO("BACKUP_EXCLUDE", "Exclude files with no-backup attributes")
} }
@@ -255,7 +255,8 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
go func() { go func() {
// List local files // List local files
defer CatchLogException() defer CatchLogException()
localSnapshot.ListLocalFiles(shadowTop, manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute, localListingChannel, &skippedDirectories, &skippedFiles) localSnapshot.ListLocalFiles(shadowTop, manager.options.NoBackupFile, manager.options.FiltersFile,
manager.options.ExcludeByAttribute, localListingChannel, &skippedDirectories, &skippedFiles)
}() }()
go func() { go func() {
@@ -642,7 +643,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
// Restore downloads the specified snapshot, compares it with what's on the repository, and then downloads // Restore downloads the specified snapshot, compares it with what's on the repository, and then downloads
// files that are different.'QuickMode' will bypass files with unchanged sizes and timestamps. 'DeleteMode' will // files that are different.'QuickMode' will bypass files with unchanged sizes and timestamps. 'DeleteMode' will
// remove local files that don't exist in the snapshot. 'Patterns' is used to include/exclude certain files. // remove local files that don't exist in the snapshot. 'Patterns' is used to include/exclude certain files.
func (manager *BackupManager) Restore(top string, revision int, options RestoreOptions) int { func (manager *BackupManager) Restore(top string, revision int, options *RestoreOptions) int {
if options.Threads < 1 { if options.Threads < 1 {
options.Threads = 1 options.Threads = 1
} }
@@ -653,10 +654,10 @@ func (manager *BackupManager) Restore(top string, revision int, options RestoreO
allowFailures := options.AllowFailures allowFailures := options.AllowFailures
metadataOptions := RestoreMetadataOptions{ metadataOptions := RestoreMetadataOptions{
SetOwner: options.SetOwner, SetOwner: manager.options.SetOwner,
ExcludeXattrs: options.ExcludeXattrs, ExcludeXattrs: manager.options.ExcludeXattrs,
NormalizeXattrs: options.NormalizeXattrs, NormalizeXattrs: manager.options.NormalizeXattrs,
FileFlagsMask: options.FileFlagsMask, FileFlagsMask: manager.options.FileFlagsMask,
} }
startTime := time.Now().Unix() startTime := time.Now().Unix()
@@ -715,7 +716,8 @@ func (manager *BackupManager) Restore(top string, revision int, options RestoreO
go func() { go func() {
// List local files // List local files
defer CatchLogException() defer CatchLogException()
localSnapshot.ListLocalFiles(top, manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute, localListingChannel, nil, nil) localSnapshot.ListLocalFiles(top, manager.options.NoBackupFile, manager.options.FiltersFile,
manager.options.ExcludeByAttribute, localListingChannel, nil, nil)
}() }()
remoteSnapshot := manager.SnapshotManager.DownloadSnapshot(manager.snapshotID, revision) remoteSnapshot := manager.SnapshotManager.DownloadSnapshot(manager.snapshotID, revision)
@@ -857,12 +859,12 @@ func (manager *BackupManager) Restore(top string, revision int, options RestoreO
return 0 return 0
} }
} }
err = remoteEntry.RestoreEarlyDirFlags(fullPath, options.FileFlagsMask) err = remoteEntry.RestoreEarlyDirFlags(fullPath, manager.options.FileFlagsMask)
if err != nil { if err != nil {
LOG_WARN("DOWNLOAD_FLAGS", "Failed to set early file flags on %s: %v", fullPath, err) LOG_WARN("DOWNLOAD_FLAGS", "Failed to set early file flags on %s: %v", fullPath, err)
} }
directoryEntries = append(directoryEntries, remoteEntry) directoryEntries = append(directoryEntries, remoteEntry)
} else if remoteEntry.IsSpecial() && options.IncludeSpecials { } else if remoteEntry.IsSpecial() && manager.options.IncludeSpecials {
if stat, _ := os.Lstat(fullPath); stat != nil { if stat, _ := os.Lstat(fullPath); stat != nil {
if remoteEntry.IsSameSpecial(stat) { if remoteEntry.IsSameSpecial(stat) {
remoteEntry.RestoreMetadata(fullPath, nil, metadataOptions) remoteEntry.RestoreMetadata(fullPath, nil, metadataOptions)

View File

@@ -251,21 +251,20 @@ func TestBackupManager(t *testing.T) {
time.Sleep(time.Duration(delay) * time.Second) time.Sleep(time.Duration(delay) * time.Second)
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
backupManager := CreateBackupManager("host1", storage, testDir, password, "", "", false) backupManager := CreateBackupManager("host1", storage, testDir, password, nil)
backupManager.SetupSnapshotCache("default") backupManager.SetupSnapshotCache("default")
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 1024, 1024) backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 1024, 1024)
time.Sleep(time.Duration(delay) * time.Second) time.Sleep(time.Duration(delay) * time.Second)
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
failedFiles := backupManager.Restore(testDir+"/repository2", 1, RestoreOptions{ failedFiles := backupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: nil, Patterns: nil,
InPlace: false, InPlace: false,
QuickMode: false, QuickMode: false,
Overwrite: true, Overwrite: true,
DeleteMode: false, DeleteMode: false,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: false, AllowFailures: false,
}) })
@@ -292,14 +291,13 @@ func TestBackupManager(t *testing.T) {
backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false, 0, false, 1024, 1024) backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false, 0, false, 1024, 1024)
time.Sleep(time.Duration(delay) * time.Second) time.Sleep(time.Duration(delay) * time.Second)
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
failedFiles = backupManager.Restore(testDir+"/repository2", 2, RestoreOptions{ failedFiles = backupManager.Restore(testDir+"/repository2", 2, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: nil, Patterns: nil,
InPlace: true, InPlace: true,
QuickMode: true, QuickMode: true,
Overwrite: true, Overwrite: true,
DeleteMode: false, DeleteMode: false,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: false, AllowFailures: false,
}) })
@@ -330,14 +328,13 @@ func TestBackupManager(t *testing.T) {
createRandomFile(testDir+"/repository2/dir5/file5", 100) createRandomFile(testDir+"/repository2/dir5/file5", 100)
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
failedFiles = backupManager.Restore(testDir+"/repository2", 3, RestoreOptions{ failedFiles = backupManager.Restore(testDir+"/repository2", 3, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: nil, Patterns: nil,
InPlace: true, InPlace: true,
QuickMode: false, QuickMode: false,
Overwrite: true, Overwrite: true,
DeleteMode: true, DeleteMode: true,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: false, AllowFailures: false,
}) })
@@ -367,14 +364,13 @@ func TestBackupManager(t *testing.T) {
os.Remove(testDir + "/repository1/file2") os.Remove(testDir + "/repository1/file2")
os.Remove(testDir + "/repository1/dir1/file3") os.Remove(testDir + "/repository1/dir1/file3")
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
failedFiles = backupManager.Restore(testDir+"/repository1", 3, RestoreOptions{ failedFiles = backupManager.Restore(testDir+"/repository1", 3, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: []string{"+file2", "+dir1/file3", "-*"}, Patterns: []string{"+file2", "+dir1/file3", "-*"},
InPlace: true, InPlace: true,
QuickMode: false, QuickMode: false,
Overwrite: true, Overwrite: true,
DeleteMode: false, DeleteMode: false,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: false, AllowFailures: false,
}) })
@@ -564,7 +560,7 @@ func TestPersistRestore(t *testing.T) {
// do unencrypted backup // do unencrypted backup
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
unencBackupManager := CreateBackupManager("host1", unencStorage, testDir, "", "", "", false) unencBackupManager := CreateBackupManager("host1", unencStorage, testDir, "", nil)
unencBackupManager.SetupSnapshotCache("default") unencBackupManager.SetupSnapshotCache("default")
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
@@ -573,7 +569,7 @@ func TestPersistRestore(t *testing.T) {
// do encrypted backup // do encrypted backup
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
encBackupManager := CreateBackupManager("host1", storage, testDir, password, "", "", false) encBackupManager := CreateBackupManager("host1", storage, testDir, password, nil)
encBackupManager.SetupSnapshotCache("default") encBackupManager.SetupSnapshotCache("default")
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
@@ -651,14 +647,13 @@ func TestPersistRestore(t *testing.T) {
// test restore all uncorrupted to repository3 // test restore all uncorrupted to repository3
SetDuplicacyPreferencePath(testDir + "/repository3/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository3/.duplicacy")
failedFiles := unencBackupManager.Restore(testDir+"/repository3", 1, RestoreOptions{ failedFiles := unencBackupManager.Restore(testDir+"/repository3", 1, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: nil, Patterns: nil,
InPlace: true, InPlace: true,
QuickMode: false, QuickMode: false,
Overwrite: false, Overwrite: false,
DeleteMode: false, DeleteMode: false,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: false, AllowFailures: false,
}) })
@@ -706,14 +701,13 @@ func TestPersistRestore(t *testing.T) {
// test restore corrupted, inPlace = true, corrupted files will have hash failures // test restore corrupted, inPlace = true, corrupted files will have hash failures
os.RemoveAll(testDir + "/repository2") os.RemoveAll(testDir + "/repository2")
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
failedFiles = unencBackupManager.Restore(testDir+"/repository2", 1, RestoreOptions{ failedFiles = unencBackupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: nil, Patterns: nil,
InPlace: true, InPlace: true,
QuickMode: false, QuickMode: false,
Overwrite: false, Overwrite: false,
DeleteMode: false, DeleteMode: false,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: true, AllowFailures: true,
}) })
@@ -724,14 +718,13 @@ func TestPersistRestore(t *testing.T) {
os.RemoveAll(testDir + "/repository2") os.RemoveAll(testDir + "/repository2")
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
failedFiles = encBackupManager.Restore(testDir+"/repository2", 1, RestoreOptions{ failedFiles = encBackupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: nil, Patterns: nil,
InPlace: true, InPlace: true,
QuickMode: false, QuickMode: false,
Overwrite: false, Overwrite: false,
DeleteMode: false, DeleteMode: false,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: true, AllowFailures: true,
}) })
@@ -744,14 +737,13 @@ func TestPersistRestore(t *testing.T) {
// test restore corrupted, inPlace = false, corrupted files will be missing // test restore corrupted, inPlace = false, corrupted files will be missing
os.RemoveAll(testDir + "/repository2") os.RemoveAll(testDir + "/repository2")
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
failedFiles = unencBackupManager.Restore(testDir+"/repository2", 1, RestoreOptions{ failedFiles = unencBackupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: nil, Patterns: nil,
InPlace: false, InPlace: false,
QuickMode: false, QuickMode: false,
Overwrite: false, Overwrite: false,
DeleteMode: false, DeleteMode: false,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: true, AllowFailures: true,
}) })
@@ -762,14 +754,13 @@ func TestPersistRestore(t *testing.T) {
os.RemoveAll(testDir + "/repository2") os.RemoveAll(testDir + "/repository2")
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
failedFiles = encBackupManager.Restore(testDir+"/repository2", 1, RestoreOptions{ failedFiles = encBackupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: nil, Patterns: nil,
InPlace: false, InPlace: false,
QuickMode: false, QuickMode: false,
Overwrite: false, Overwrite: false,
DeleteMode: false, DeleteMode: false,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: true, AllowFailures: true,
}) })
@@ -782,27 +773,25 @@ func TestPersistRestore(t *testing.T) {
// with overwrite=true, corrupted file1 from unenc will be restored correctly from enc // with overwrite=true, corrupted file1 from unenc will be restored correctly from enc
// the latter will not touch the existing file3 with correct hash // the latter will not touch the existing file3 with correct hash
os.RemoveAll(testDir + "/repository2") os.RemoveAll(testDir + "/repository2")
failedFiles = unencBackupManager.Restore(testDir+"/repository2", 1, RestoreOptions{ failedFiles = unencBackupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: nil, Patterns: nil,
InPlace: true, InPlace: true,
QuickMode: false, QuickMode: false,
Overwrite: false, Overwrite: false,
DeleteMode: false, DeleteMode: false,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: true, AllowFailures: true,
}) })
assertRestoreFailures(t, failedFiles, 1) assertRestoreFailures(t, failedFiles, 1)
failedFiles = encBackupManager.Restore(testDir+"/repository2", 1, RestoreOptions{ failedFiles = encBackupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: nil, Patterns: nil,
InPlace: true, InPlace: true,
QuickMode: false, QuickMode: false,
Overwrite: true, Overwrite: true,
DeleteMode: false, DeleteMode: false,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: true, AllowFailures: true,
}) })
@@ -812,28 +801,26 @@ func TestPersistRestore(t *testing.T) {
// restore to repository3, with overwrite and allowFailures (true/false), quickMode = false (use hashes) // restore to repository3, with overwrite and allowFailures (true/false), quickMode = false (use hashes)
// should always succeed as uncorrupted files already exist with correct hash, so these will be ignored // should always succeed as uncorrupted files already exist with correct hash, so these will be ignored
SetDuplicacyPreferencePath(testDir + "/repository3/.duplicacy") SetDuplicacyPreferencePath(testDir + "/repository3/.duplicacy")
failedFiles = unencBackupManager.Restore(testDir+"/repository3", 1, RestoreOptions{ failedFiles = unencBackupManager.Restore(testDir+"/repository3", 1, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: nil, Patterns: nil,
InPlace: true, InPlace: true,
QuickMode: false, QuickMode: false,
Overwrite: true, Overwrite: true,
DeleteMode: false, DeleteMode: false,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: false, AllowFailures: false,
}) })
assertRestoreFailures(t, failedFiles, 0) assertRestoreFailures(t, failedFiles, 0)
checkAllUncorrupted("/repository3") checkAllUncorrupted("/repository3")
failedFiles = unencBackupManager.Restore(testDir+"/repository3", 1, RestoreOptions{ failedFiles = unencBackupManager.Restore(testDir+"/repository3", 1, &RestoreOptions{
Threads: threads, Threads: threads,
Patterns: nil, Patterns: nil,
InPlace: true, InPlace: true,
QuickMode: false, QuickMode: false,
Overwrite: true, Overwrite: true,
DeleteMode: false, DeleteMode: false,
SetOwner: false,
ShowStatistics: false, ShowStatistics: false,
AllowFailures: true, AllowFailures: true,
}) })

View File

@@ -50,6 +50,7 @@ type Preference struct {
NobackupFile string `json:"nobackup_file"` NobackupFile string `json:"nobackup_file"`
Keys map[string]string `json:"keys"` Keys map[string]string `json:"keys"`
FiltersFile string `json:"filters"` FiltersFile string `json:"filters"`
ExcludeOwner bool `json:"exclude_owner"`
ExcludeByAttribute bool `json:"exclude_by_attribute"` ExcludeByAttribute bool `json:"exclude_by_attribute"`
ExcludeXattrs bool `json:"exclude_xattrs"` ExcludeXattrs bool `json:"exclude_xattrs"`
NormalizeXattrs bool `json:"normalize_xattrs"` NormalizeXattrs bool `json:"normalize_xattrs"`