diff --git a/README.md b/README.md index a1c38b3..188f672 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Dupluxe +# Dupluxy An experimental Duplicacy derivative with improved support for preserving state on UNIX like systems. Produces snapshots compatible with Duplicacy. diff --git a/dupluxy/dupluxy.go b/dupluxy/dupluxy.go new file mode 100644 index 0000000..9c5602f --- /dev/null +++ b/dupluxy/dupluxy.go @@ -0,0 +1,2314 @@ +// Copyright (c) Acrosync LLC. All rights reserved. +// Free for personal use and commercial trial +// Commercial use requires per-user licenses available from https://duplicacy.com + +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "os/exec" + "os/signal" + "path" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + + _ "net/http/pprof" + + "github.com/gilbertchen/cli" + + duplicacy "github.com/dupluxy/dupluxy/src" +) + +const ( + ArgumentExitCode = 3 +) + +var ScriptEnabled bool +var Version = "0.1.0" +var GitCommit = "unofficial" + +func getRepositoryPreference(context *cli.Context, storageName string) (repository string, + preference *duplicacy.Preference) { + + repository, err := os.Getwd() + if err != nil { + duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to retrieve the current working directory: %v", err) + return "", nil + } + + for { + stat, err := os.Stat(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY)) //TOKEEP + if err != nil && !os.IsNotExist(err) { + duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to retrieve the information about the directory %s: %v", + repository, err) + return "", nil + } + + if stat != nil && (stat.IsDir() || stat.Mode().IsRegular()) { + break + } + + parent := path.Dir(repository) + if parent == repository || parent == "" { + duplicacy.LOG_ERROR("REPOSITORY_PATH", "Repository has not been initialized") + return "", nil + } + repository = parent + } + duplicacy.LoadPreferences(repository) + + preferencePath := duplicacy.GetDuplicacyPreferencePath() + duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring")) + + if storageName == "" { + storageName = context.String("storage") + } + + if storageName == "" { + if duplicacy.Preferences[0].RepositoryPath != "" { + repository = duplicacy.Preferences[0].RepositoryPath + duplicacy.LOG_INFO("REPOSITORY_SET", "Repository set to %s", repository) + } + return repository, &duplicacy.Preferences[0] + } + + preference = duplicacy.FindPreference(storageName) + + if preference == nil { + duplicacy.LOG_ERROR("STORAGE_NONE", "No storage named '%s' is found", storageName) + return "", nil + } + + if preference.RepositoryPath != "" { + repository = preference.RepositoryPath + duplicacy.LOG_INFO("REPOSITORY_SET", "Repository set to %s", repository) + } + + return repository, preference +} + +func getRevisions(context *cli.Context) (revisions []int) { + + flags := context.StringSlice("r") + + rangeRegex := regexp.MustCompile(`^([0-9]+)-([0-9]+)$`) + numberRegex := regexp.MustCompile(`^([0-9]+)$`) + + for _, flag := range flags { + matched := rangeRegex.FindStringSubmatch(flag) + if matched != nil { + start, _ := strconv.Atoi(matched[1]) + end, _ := strconv.Atoi(matched[2]) + if end > start { + for r := start; r <= end; r++ { + revisions = append(revisions, r) + } + continue + } + } + + matched = numberRegex.FindStringSubmatch(flag) + if matched != nil { + r, _ := strconv.Atoi(matched[1]) + revisions = append(revisions, r) + continue + } + + fmt.Fprintf(context.App.Writer, "Invalid revision: %s.\n\n", flag) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + return revisions + +} + +func setGlobalOptions(context *cli.Context) { + if context.GlobalBool("log") { + duplicacy.EnableLogHeader() + } + + if context.GlobalBool("stack") { + duplicacy.EnableStackTrace() + } + + if context.GlobalBool("verbose") { + duplicacy.SetLoggingLevel(duplicacy.TRACE) + } + + if context.GlobalBool("debug") { + duplicacy.SetLoggingLevel(duplicacy.DEBUG) + } + + if context.GlobalBool("print-memory-usage") { + go duplicacy.PrintMemoryUsage() + } + + ScriptEnabled = true + if context.GlobalBool("no-script") { + ScriptEnabled = false + } + + address := context.GlobalString("profile") + if address != "" { + go func() { + http.ListenAndServe(address, nil) + }() + } + + for _, logID := range context.GlobalStringSlice("suppress") { + duplicacy.SuppressLog(logID) + } + + duplicacy.RunInBackground = context.GlobalBool("background") +} + +func runScript(context *cli.Context, storageName string, phase string) bool { + + if !ScriptEnabled { + return false + } + + preferencePath := duplicacy.GetDuplicacyPreferencePath() + scriptDir, _ := filepath.Abs(path.Join(preferencePath, "scripts")) + scriptNames := []string{phase + "-" + context.Command.Name, + storageName + "-" + phase + "-" + context.Command.Name} + + script := "" + for _, scriptName := range scriptNames { + script = path.Join(scriptDir, scriptName) + if runtime.GOOS == "windows" { + script += ".bat" + } + if _, err := os.Stat(script); err == nil { + break + } else { + script = "" + } + } + + if script == "" { + return false + } + + duplicacy.LOG_INFO("SCRIPT_RUN", "Running script %s", script) + + output, err := exec.Command(script, os.Args...).CombinedOutput() + for _, line := range strings.Split(string(output), "\n") { + line := strings.TrimSpace(line) + if line != "" { + duplicacy.LOG_INFO("SCRIPT_OUTPUT", line) + } + } + + if err != nil { + duplicacy.LOG_ERROR("SCRIPT_ERROR", "Failed to run %s script: %v", script, err) + return false + } + + return true +} + +func loadRSAPrivateKey(keyFile string, passphrase string, preference *duplicacy.Preference, backupManager *duplicacy.BackupManager, resetPasswords bool) { + if keyFile == "" { + return + } + + prompt := fmt.Sprintf("Enter the passphrase for %s:", keyFile) + if passphrase == "" { + passphrase = duplicacy.GetPassword(*preference, "rsa_passphrase", prompt, false, resetPasswords) + backupManager.LoadRSAPrivateKey(keyFile, passphrase) + duplicacy.SavePassword(*preference, "rsa_passphrase", passphrase) + } else { + backupManager.LoadRSAPrivateKey(keyFile, passphrase) + } + +} + +func initRepository(context *cli.Context) { + configRepository(context, true) +} + +func addStorage(context *cli.Context) { + configRepository(context, false) +} + +func configRepository(context *cli.Context, init bool) { + + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + numberOfArgs := 3 + if init { + numberOfArgs = 2 + } + if len(context.Args()) != numberOfArgs { + fmt.Fprintf(context.App.Writer, "The %s command requires %d arguments.\n\n", + context.Command.Name, numberOfArgs) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + var storageName string + var snapshotID string + var storageURL string + + if init { + storageName = context.String("storage-name") + if len(storageName) == 0 { + storageName = "default" + } + snapshotID = context.Args()[0] + storageURL = context.Args()[1] + } else { + storageName = context.Args()[0] + snapshotID = context.Args()[1] + storageURL = context.Args()[2] + + if strings.ToLower(storageName) == "ssh" { + duplicacy.LOG_ERROR("PREFERENCE_INVALID", "'%s' is an invalid storage name", storageName) + return + } + } + + snapshotIDRegex := regexp.MustCompile(`^[A-Za-z0-9_\-]+$`) + matched := snapshotIDRegex.FindStringSubmatch(snapshotID) + if matched == nil { + duplicacy.LOG_ERROR("PREFERENCE_INVALID", "'%s' is an invalid snapshot id", snapshotID) + return + } + + var repository string + var err error + + if init { + repository, err = os.Getwd() + if err != nil { + duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to retrieve the current working directory: %v", err) + return + } + + preferencePath := context.String("pref-dir") + if preferencePath == "" { + preferencePath = path.Join(repository, duplicacy.DUPLICACY_DIRECTORY) // TOKEEP + } + + if stat, _ := os.Stat(path.Join(preferencePath, "preferences")); stat != nil { + duplicacy.LOG_ERROR("REPOSITORY_INIT", "The repository %s has already been initialized", repository) + return + } + + err = os.Mkdir(preferencePath, 0744) + if err != nil && !os.IsExist(err) { + duplicacy.LOG_ERROR("REPOSITORY_INIT", "Failed to create the directory %s: %v", + preferencePath, err) + return + } + if context.String("pref-dir") != "" { + // out of tree preference file + // write real path into .duplicacy file inside repository + duplicacyFileName := path.Join(repository, duplicacy.DUPLICACY_FILE) + d1 := []byte(preferencePath) + err = os.WriteFile(duplicacyFileName, d1, 0644) + if err != nil { + duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to write %s file inside repository %v", duplicacyFileName, err) + return + } + } + duplicacy.SetDuplicacyPreferencePath(preferencePath) + duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring")) + + } else { + repository, _ = getRepositoryPreference(context, "") + if duplicacy.FindPreference(storageName) != nil { + duplicacy.LOG_ERROR("STORAGE_DUPLICATE", "There is already a storage named '%s'", storageName) + return + } + } + + repositoryPath := "" + if context.String("repository") != "" { + repositoryPath = context.String("repository") + } + preference := duplicacy.Preference{ + Name: storageName, + SnapshotID: snapshotID, + RepositoryPath: repositoryPath, + StorageURL: storageURL, + Encrypted: context.Bool("encrypt"), + } + + storage := duplicacy.CreateStorage(preference, true, 1) + storagePassword := "" + if preference.Encrypted { + prompt := fmt.Sprintf("Enter storage password for %s:", preference.StorageURL) + storagePassword = duplicacy.GetPassword(preference, "password", prompt, false, true) + } else { + if context.String("key") != "" { + duplicacy.LOG_ERROR("STORAGE_CONFIG", "RSA encryption can't be enabled with an unencrypted storage") + return + } + } + + existingConfig, _, err := duplicacy.DownloadConfig(storage, storagePassword) + if err != nil { + duplicacy.LOG_ERROR("STORAGE_CONFIG", "Failed to download the configuration file from the storage: %v", err) + return + } + + if existingConfig != nil { + duplicacy.LOG_INFO("STORAGE_CONFIGURED", + "The storage '%s' has already been initialized", preference.StorageURL) + if existingConfig.CompressionLevel >= -1 && existingConfig.CompressionLevel <= 9 { + duplicacy.LOG_INFO("STORAGE_FORMAT", "This storage is configured to use the pre-1.2.0 format") + } else if existingConfig.CompressionLevel != duplicacy.DEFAULT_COMPRESSION_LEVEL { + duplicacy.LOG_INFO("STORAGE_COMPRESSION", "Compression level: %d", existingConfig.CompressionLevel) + } + + // Don't print config in the background mode + if !duplicacy.RunInBackground { + existingConfig.Print() + } + } else { + averageChunkSize := duplicacy.AtoSize(context.String("chunk-size")) + if averageChunkSize == 0 { + fmt.Fprintf(context.App.Writer, "Invalid average chunk size: %s.\n\n", context.String("chunk-size")) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + size := 1 + for size*2 <= averageChunkSize { + size *= 2 + } + + if size != averageChunkSize { + fmt.Fprintf(context.App.Writer, "Invalid average chunk size: %d is not a power of 2.\n\n", + averageChunkSize) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + maximumChunkSize := 4 * averageChunkSize + minimumChunkSize := averageChunkSize / 4 + + if context.String("max-chunk-size") != "" { + maximumChunkSize = duplicacy.AtoSize(context.String("max-chunk-size")) + if maximumChunkSize < averageChunkSize { + fmt.Fprintf(context.App.Writer, "Invalid maximum chunk size: %s.\n\n", + context.String("max-chunk-size")) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + } + + if context.String("min-chunk-size") != "" { + minimumChunkSize = duplicacy.AtoSize(context.String("min-chunk-size")) + if minimumChunkSize > averageChunkSize || minimumChunkSize == 0 { + fmt.Fprintf(context.App.Writer, "Invalid minimum chunk size: %s.\n\n", + context.String("min-chunk-size")) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + } + + if preference.Encrypted { + repeatedPassword := duplicacy.GetPassword(preference, "password", "Re-enter storage password:", + false, true) + if repeatedPassword != storagePassword { + duplicacy.LOG_ERROR("STORAGE_PASSWORD", "Storage passwords do not match") + return + } + } + + var otherConfig *duplicacy.Config + var bitCopy bool + if context.String("copy") != "" { + + otherPreference := duplicacy.FindPreference(context.String("copy")) + + if otherPreference == nil { + duplicacy.LOG_ERROR("STORAGE_NOTFOUND", "Storage '%s' can't be found", context.String("copy")) + return + } + + otherStorage := duplicacy.CreateStorage(*otherPreference, false, 1) + + otherPassword := "" + if otherPreference.Encrypted { + prompt := fmt.Sprintf("Enter storage password for %s:", otherPreference.StorageURL) + otherPassword = duplicacy.GetPassword(*otherPreference, "password", prompt, false, false) + } + + otherConfig, _, err = duplicacy.DownloadConfig(otherStorage, otherPassword) + if err != nil { + duplicacy.LOG_ERROR("STORAGE_COPY", "Failed to download the configuration file from the storage: %v", + err) + return + } + + if otherConfig == nil { + duplicacy.LOG_ERROR("STORAGE_NOT_CONFIGURED", + "The storage to copy the configuration from has not been initialized") + } + + bitCopy = context.Bool("bit-identical") + } + + iterations := context.Int("iterations") + if iterations == 0 { + iterations = duplicacy.CONFIG_DEFAULT_ITERATIONS + } + + dataShards := 0 + parityShards := 0 + shards := context.String("erasure-coding") + if shards != "" { + shardsRegex := regexp.MustCompile(`^([0-9]+):([0-9]+)$`) + matched := shardsRegex.FindStringSubmatch(shards) + if matched == nil { + duplicacy.LOG_ERROR("STORAGE_ERASURECODE", "Invalid erasure coding parameters: %s", shards) + } else { + dataShards, _ = strconv.Atoi(matched[1]) + parityShards, _ = strconv.Atoi(matched[2]) + if dataShards == 0 || dataShards > 256 || parityShards == 0 || parityShards > dataShards { + duplicacy.LOG_ERROR("STORAGE_ERASURECODE", "Invalid erasure coding parameters: %s", shards) + } + } + } + + compressionLevel := 100 + zstdLevel := context.String("zstd-level") + if zstdLevel != "" { + if level, found := duplicacy.ZSTD_COMPRESSION_LEVELS[zstdLevel]; found { + compressionLevel = level + } else { + duplicacy.LOG_ERROR("STORAGE_COMPRESSION", "Invalid zstd compression level: %s", zstdLevel) + } + } else if context.Bool("zstd") { + compressionLevel = duplicacy.ZSTD_COMPRESSION_LEVEL_DEFAULT + } + + duplicacy.ConfigStorage(storage, iterations, compressionLevel, averageChunkSize, maximumChunkSize, + minimumChunkSize, storagePassword, otherConfig, bitCopy, context.String("key"), dataShards, parityShards) + } + + duplicacy.Preferences = append(duplicacy.Preferences, preference) + + duplicacy.SavePreferences() + + if repositoryPath == "" { + repositoryPath = repository + } + + duplicacy.LOG_INFO("REPOSITORY_INIT", "%s will be backed up to %s with id %s", + repositoryPath, preference.StorageURL, preference.SnapshotID) +} + +type TriBool struct { + Value int +} + +func (triBool *TriBool) Set(value string) error { + value = strings.ToLower(value) + if value == "yes" || value == "true" || value == "1" { + triBool.Value = 2 + return nil + } else if value == "no" || value == "false" || value == "0" { + triBool.Value = 1 + return nil + } else if value == "" { + // Only set to true if it hasn't been set before. This is necessary because for 'encrypt, e' this may + // be called twice, the second time with a value of "" + if triBool.Value == 0 { + triBool.Value = 2 + } + return nil + } else { + return fmt.Errorf("Invalid boolean value '%s'", value) + } +} + +// IsBoolFlag implements the private interface flag.boolFlag to indicate that this is a bool flag so the argument +// is optional +func (triBool *TriBool) IsBoolFlag() bool { return true } + +func (triBool *TriBool) String() string { + return "" +} + +func (triBool *TriBool) IsSet() bool { + return triBool.Value != 0 +} + +func (triBool *TriBool) IsTrue() bool { + return triBool.Value == 2 +} + +func setPreference(context *cli.Context) { + + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + if len(context.Args()) > 0 { + fmt.Fprintf(context.App.Writer, "The %s command takes no arguments.\n\n", context.Command.Name) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + storageName := context.String("storage") + + repository, oldPreference := getRepositoryPreference(context, storageName) + + if oldPreference == nil { + duplicacy.LOG_ERROR("STORAGE_SET", "The storage '%s' has not been added to the repository %s", + storageName, repository) + return + } + + newPreference := *oldPreference + + triBool := context.Generic("e").(*TriBool) + if triBool.IsSet() { + newPreference.Encrypted = triBool.IsTrue() + } + + triBool = context.Generic("no-backup").(*TriBool) + if triBool.IsSet() { + newPreference.BackupProhibited = triBool.IsTrue() + } + + triBool = context.Generic("no-restore").(*TriBool) + if triBool.IsSet() { + newPreference.RestoreProhibited = triBool.IsTrue() + } + + triBool = context.Generic("no-save-password").(*TriBool) + if triBool.IsSet() { + newPreference.DoNotSavePassword = triBool.IsTrue() + } + + if context.String("nobackup-file") != "" { + newPreference.NobackupFile = context.String("nobackup-file") + } + + if context.String("filters") != "" { + newPreference.FiltersFile = context.String("filters") + } + + triBool = context.Generic("exclude-by-attribute").(*TriBool) + if triBool.IsSet() { + newPreference.ExcludeByAttribute = triBool.IsTrue() + } + + key := context.String("key") + value := context.String("value") + + if len(key) > 0 { + + // Make a deep copy of the keys otherwise we would be change both preferences at once. + newKeys := make(map[string]string) + for k, v := range newPreference.Keys { + newKeys[k] = v + } + newPreference.Keys = newKeys + + if len(value) == 0 { + delete(newPreference.Keys, key) + } else { + if len(newPreference.Keys) == 0 { + newPreference.Keys = make(map[string]string) + } + newPreference.Keys[key] = value + } + } + + if duplicacy.IsTracing() { + description, _ := json.MarshalIndent(newPreference, "", " ") + fmt.Printf("%s\n", description) + } + + if newPreference.Equal(oldPreference) { + duplicacy.LOG_INFO("STORAGE_SET", "The options for storage %s have not been modified", + oldPreference.StorageURL) + } else { + *oldPreference = newPreference + duplicacy.SavePreferences() + duplicacy.LOG_INFO("STORAGE_SET", "New options for storage %s have been saved", oldPreference.StorageURL) + } +} + +func changePassword(context *cli.Context) { + + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + if len(context.Args()) != 0 { + fmt.Fprintf(context.App.Writer, "The %s command requires no arguments.\n\n", + context.Command.Name) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + _, preference := getRepositoryPreference(context, "") + + storage := duplicacy.CreateStorage(*preference, false, 1) + if storage == nil { + return + } + + password := "" + if preference.Encrypted { + password = duplicacy.GetPassword(*preference, "password", + fmt.Sprintf("Enter old password for storage %s:", preference.StorageURL), + false, true) + } + + config, _, err := duplicacy.DownloadConfig(storage, password) + if err != nil { + duplicacy.LOG_ERROR("STORAGE_CONFIG", "Failed to download the configuration file from the storage: %v", err) + return + } + + if config == nil { + duplicacy.LOG_ERROR("STORAGE_NOT_CONFIGURED", "The storage has not been initialized") + return + } + + newPassword := duplicacy.GetPassword(*preference, "password", "Enter new storage password:", false, true) + repeatedPassword := duplicacy.GetPassword(*preference, "password", "Re-enter new storage password:", false, true) + if repeatedPassword != newPassword { + duplicacy.LOG_ERROR("PASSWORD_CHANGE", "The new passwords do not match") + return + } + if newPassword == password { + duplicacy.LOG_ERROR("PASSWORD_CHANGE", "The new password is the same as the old one") + return + } + + iterations := context.Int("iterations") + if iterations == 0 { + iterations = duplicacy.CONFIG_DEFAULT_ITERATIONS + } + + description, err := json.MarshalIndent(config, "", " ") + if err != nil { + duplicacy.LOG_ERROR("CONFIG_MARSHAL", "Failed to marshal the config: %v", err) + return + } + + configPath := path.Join(duplicacy.GetDuplicacyPreferencePath(), "config") + err = os.WriteFile(configPath, description, 0600) + if err != nil { + duplicacy.LOG_ERROR("CONFIG_SAVE", "Failed to save the old config to %s: %v", configPath, err) + return + } + duplicacy.LOG_INFO("CONFIG_SAVE", "The old config has been temporarily saved to %s", configPath) + + removeLocalCopy := false + defer func() { + if removeLocalCopy { + err = os.Remove(configPath) + if err != nil { + duplicacy.LOG_WARN("CONFIG_CLEAN", "Failed to delete %s: %v", configPath, err) + } else { + duplicacy.LOG_INFO("CONFIG_CLEAN", "The local copy of the old config has been removed") + } + } + }() + + err = storage.DeleteFile(0, "config") + if err != nil { + duplicacy.LOG_ERROR("CONFIG_DELETE", "Failed to delete the old config from the storage: %v", err) + return + } + + duplicacy.UploadConfig(storage, config, newPassword, iterations) + + duplicacy.SavePassword(*preference, "password", newPassword) + + duplicacy.LOG_INFO("STORAGE_SET", "The password for storage %s has been changed", preference.StorageURL) + + removeLocalCopy = true +} + +func backupRepository(context *cli.Context) { + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + if len(context.Args()) != 0 { + fmt.Fprintf(context.App.Writer, "The %s command requires no arguments.\n\n", context.Command.Name) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + repository, preference := getRepositoryPreference(context, "") + + if preference.BackupProhibited { + duplicacy.LOG_ERROR("BACKUP_DISABLED", "Backup from this repository to %s was disabled by the preference", + preference.StorageURL) + return + } + + runScript(context, preference.Name, "pre") + + threads := context.Int("threads") + if threads < 1 { + threads = 1 + } + + duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL) + storage := duplicacy.CreateStorage(*preference, false, threads) + if storage == nil { + return + } + + password := "" + if preference.Encrypted { + password = duplicacy.GetPassword(*preference, "password", "Enter storage password:", false, false) + } + + quickMode := true + if context.Bool("hash") { + quickMode = false + } + + showStatistics := context.Bool("stats") + + enableVSS := context.Bool("vss") + vssTimeout := context.Int("vss-timeout") + + dryRun := context.Bool("dry-run") + uploadRateLimit := context.Int("limit-rate") + enumOnly := context.Bool("enum-only") + storage.SetRateLimits(0, uploadRateLimit) + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, + &duplicacy.BackupManagerOptions{ + NobackupFile: preference.NobackupFile, + FiltersFile: preference.FiltersFile, + ExcludeByAttribute: preference.ExcludeByAttribute, + ExcludeXattrs: preference.ExcludeXattrs, + NormalizeXattrs: preference.NormalizeXattrs, + IncludeFileFlags: preference.IncludeFileFlags, + IncludeSpecials: preference.IncludeSpecials, + FileFlagsMask: uint32(preference.FileFlagsMask), + }) + duplicacy.SavePassword(*preference, "password", password) + + backupManager.SetupSnapshotCache(preference.Name) + backupManager.SetDryRun(dryRun) + + zstdLevel := context.String("zstd-level") + if zstdLevel != "" { + if level, found := duplicacy.ZSTD_COMPRESSION_LEVELS[zstdLevel]; found { + backupManager.SetCompressionLevel(level) + } else { + duplicacy.LOG_ERROR("STORAGE_COMPRESSION", "Invalid zstd compression level: %s", zstdLevel) + } + } else if context.Bool("zstd") { + backupManager.SetCompressionLevel(duplicacy.ZSTD_COMPRESSION_LEVEL_DEFAULT) + } + + metadataChunkSize := context.Int("metadata-chunk-size") + maximumInMemoryEntries := context.Int("max-in-memory-entries") + backupManager.Backup(repository, quickMode, threads, context.String("t"), showStatistics, enableVSS, vssTimeout, enumOnly, metadataChunkSize, maximumInMemoryEntries) + + runScript(context, preference.Name, "post") +} + +func restoreRepository(context *cli.Context) { + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + revision := context.Int("r") + if revision <= 0 { + fmt.Fprintf(context.App.Writer, "The revision flag is not specified or invalid\n\n") + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + repository, preference := getRepositoryPreference(context, "") + + if preference.RestoreProhibited { + duplicacy.LOG_ERROR("RESTORE_DISABLED", "Restore from %s to this repository was disabled by the preference", + preference.StorageURL) + return + } + + runScript(context, preference.Name, "pre") + + threads := context.Int("threads") + if threads < 1 { + threads = 1 + } + + duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL) + storage := duplicacy.CreateStorage(*preference, false, threads) + if storage == nil { + return + } + + password := "" + if preference.Encrypted { + password = duplicacy.GetPassword(*preference, "password", "Enter storage password:", false, false) + } + + var patterns []string + for _, pattern := range context.Args() { + + pattern = strings.TrimSpace(pattern) + + for strings.HasPrefix(pattern, "--") { + pattern = pattern[1:] + } + + for strings.HasPrefix(pattern, "++") { + pattern = pattern[1:] + } + + patterns = append(patterns, pattern) + } + + patterns = duplicacy.ProcessFilterLines(patterns, make([]string, 0)) + + duplicacy.LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(duplicacy.RegexMap)) + + duplicacy.LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) + + storage.SetRateLimits(context.Int("limit-rate"), 0) + + excludeOwner := preference.ExcludeOwner + // TODO: for backward compat, eventually make them all overridable? + if context.IsSet("ignore-owner") { + excludeOwner = context.Bool("ignore-owner") + } + + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, + &duplicacy.BackupManagerOptions{ + NobackupFile: preference.NobackupFile, + FiltersFile: preference.FiltersFile, + ExcludeByAttribute: preference.ExcludeByAttribute, + SetOwner: excludeOwner, + ExcludeXattrs: preference.ExcludeXattrs, + NormalizeXattrs: preference.NormalizeXattrs, + IncludeSpecials: preference.IncludeSpecials, + FileFlagsMask: uint32(preference.FileFlagsMask), + }) + + duplicacy.SavePassword(*preference, "password", password) + + loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false) + + backupManager.SetupSnapshotCache(preference.Name) + failed := backupManager.Restore(repository, revision, &duplicacy.RestoreOptions{ + InPlace: true, + QuickMode: !context.Bool("hash"), + Overwrite: context.Bool("overwrite"), + DeleteMode: context.Bool("delete"), + ShowStatistics: context.Bool("stats"), + AllowFailures: context.Bool("persist"), + }) + if failed > 0 { + duplicacy.LOG_ERROR("RESTORE_FAIL", "%d file(s) were not restored correctly", failed) + return + } + + runScript(context, preference.Name, "post") +} + +func listSnapshots(context *cli.Context) { + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + if len(context.Args()) != 0 { + fmt.Fprintf(context.App.Writer, "The %s command requires no arguments.\n\n", context.Command.Name) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + repository, preference := getRepositoryPreference(context, "") + + duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL) + + runScript(context, preference.Name, "pre") + + resetPassword := context.Bool("reset-passwords") + storage := duplicacy.CreateStorage(*preference, resetPassword, 1) + if storage == nil { + return + } + + password := "" + if preference.Encrypted { + password = duplicacy.GetPassword(*preference, "password", "Enter storage password:", + false, resetPassword) + } + + tag := context.String("t") + revisions := getRevisions(context) + + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, + &duplicacy.BackupManagerOptions{ExcludeByAttribute: preference.ExcludeByAttribute}) + duplicacy.SavePassword(*preference, "password", password) + + id := preference.SnapshotID + if context.Bool("all") { + id = "" + } else if context.String("id") != "" { + id = context.String("id") + } + + showFiles := context.Bool("files") + showChunks := context.Bool("chunks") + + // list doesn't need to decrypt file chunks; but we need -key here so we can reset the passphrase for the private key + loadRSAPrivateKey(context.String("key"), "", preference, backupManager, resetPassword) + + backupManager.SetupSnapshotCache(preference.Name) + backupManager.SnapshotManager.ListSnapshots(id, revisions, tag, showFiles, showChunks) + + runScript(context, preference.Name, "post") +} + +func checkSnapshots(context *cli.Context) { + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + if len(context.Args()) != 0 { + fmt.Fprintf(context.App.Writer, "The %s command requires no arguments.\n\n", context.Command.Name) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + repository, preference := getRepositoryPreference(context, "") + + duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL) + + runScript(context, preference.Name, "pre") + + threads := context.Int("threads") + if threads < 1 { + threads = 1 + } + + storage := duplicacy.CreateStorage(*preference, false, threads) + if storage == nil { + return + } + + password := "" + if preference.Encrypted { + password = duplicacy.GetPassword(*preference, "password", "Enter storage password:", false, false) + } + + tag := context.String("t") + revisions := getRevisions(context) + + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil) + duplicacy.SavePassword(*preference, "password", password) + + loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false) + + id := preference.SnapshotID + if context.Bool("all") { + id = "" + } else if context.String("id") != "" { + id = context.String("id") + } + + showStatistics := context.Bool("stats") + showTabular := context.Bool("tabular") + checkFiles := context.Bool("files") + checkChunks := context.Bool("chunks") + searchFossils := context.Bool("fossils") + resurrect := context.Bool("resurrect") + rewrite := context.Bool("rewrite") + persist := context.Bool("persist") + + backupManager.SetupSnapshotCache(preference.Name) + backupManager.SnapshotManager.CheckSnapshots(id, revisions, tag, showStatistics, showTabular, checkFiles, checkChunks, searchFossils, resurrect, rewrite, threads, persist) + + runScript(context, preference.Name, "post") +} + +func printFile(context *cli.Context) { + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + if len(context.Args()) > 1 { + fmt.Fprintf(context.App.Writer, "The %s command requires at most 1 argument.\n\n", context.Command.Name) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + repository, preference := getRepositoryPreference(context, "") + + runScript(context, preference.Name, "pre") + + // Do not print out storage for this command + //duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL) + storage := duplicacy.CreateStorage(*preference, false, 1) + if storage == nil { + return + } + + password := "" + if preference.Encrypted { + password = duplicacy.GetPassword(*preference, "password", "Enter storage password:", false, false) + } + + revision := context.Int("r") + + snapshotID := preference.SnapshotID + if context.String("id") != "" { + snapshotID = context.String("id") + } + + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil) + duplicacy.SavePassword(*preference, "password", password) + + loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false) + + backupManager.SetupSnapshotCache(preference.Name) + + file := "" + if len(context.Args()) > 0 { + file = context.Args()[0] + } + backupManager.SnapshotManager.PrintFile(snapshotID, revision, file) + + runScript(context, preference.Name, "post") +} + +func diff(context *cli.Context) { + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + if len(context.Args()) > 1 { + fmt.Fprintf(context.App.Writer, "The %s command requires 0 or 1 argument.\n\n", context.Command.Name) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + repository, preference := getRepositoryPreference(context, "") + + runScript(context, preference.Name, "pre") + + duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL) + storage := duplicacy.CreateStorage(*preference, false, 1) + if storage == nil { + return + } + + password := "" + if preference.Encrypted { + password = duplicacy.GetPassword(*preference, "password", "Enter storage password:", false, false) + } + + revisions := context.IntSlice("r") + if len(revisions) > 2 { + fmt.Fprintf(context.App.Writer, "The %s command requires at most 2 revisions.\n", context.Command.Name) + os.Exit(ArgumentExitCode) + } + + snapshotID := preference.SnapshotID + if context.String("id") != "" { + snapshotID = context.String("id") + } + + path := "" + if len(context.Args()) > 0 { + path = context.Args()[0] + } + + compareByHash := context.Bool("hash") + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil) + duplicacy.SavePassword(*preference, "password", password) + + loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false) + + backupManager.SetupSnapshotCache(preference.Name) + backupManager.SnapshotManager.Diff(repository, snapshotID, revisions, path, compareByHash, + duplicacy.NewListFilesOptions(preference)) + + runScript(context, preference.Name, "post") +} + +func showHistory(context *cli.Context) { + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + if len(context.Args()) != 1 { + fmt.Fprintf(context.App.Writer, "The %s command requires 1 argument.\n\n", context.Command.Name) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + repository, preference := getRepositoryPreference(context, "") + + runScript(context, preference.Name, "pre") + + duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL) + storage := duplicacy.CreateStorage(*preference, false, 1) + if storage == nil { + return + } + + password := "" + if preference.Encrypted { + password = duplicacy.GetPassword(*preference, "password", "Enter storage password:", false, false) + } + + snapshotID := preference.SnapshotID + if context.String("id") != "" { + snapshotID = context.String("id") + } + + path := context.Args()[0] + + revisions := getRevisions(context) + showLocalHash := context.Bool("hash") + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil) + duplicacy.SavePassword(*preference, "password", password) + + backupManager.SetupSnapshotCache(preference.Name) + backupManager.SnapshotManager.ShowHistory(repository, snapshotID, revisions, path, showLocalHash) + + runScript(context, preference.Name, "post") +} + +func pruneSnapshots(context *cli.Context) { + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + if len(context.Args()) != 0 { + fmt.Fprintf(context.App.Writer, "The %s command requires no arguments.\n\n", context.Command.Name) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + threads := context.Int("threads") + if threads < 1 { + threads = 1 + } + + repository, preference := getRepositoryPreference(context, "") + + runScript(context, preference.Name, "pre") + + duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL) + storage := duplicacy.CreateStorage(*preference, false, threads) + if storage == nil { + return + } + + password := "" + if preference.Encrypted { + password = duplicacy.GetPassword(*preference, "password", "Enter storage password:", false, false) + } + + revisions := getRevisions(context) + tags := context.StringSlice("t") + retentions := context.StringSlice("keep") + selfID := preference.SnapshotID + snapshotID := preference.SnapshotID + if context.Bool("all") { + snapshotID = "" + } else if context.String("id") != "" { + snapshotID = context.String("id") + } + + ignoredIDs := context.StringSlice("ignore") + exhaustive := context.Bool("exhaustive") + exclusive := context.Bool("exclusive") + dryRun := context.Bool("dry-run") + deleteOnly := context.Bool("delete-only") + collectOnly := context.Bool("collect-only") + + if !storage.IsMoveFileImplemented() && !exclusive { + fmt.Fprintf(context.App.Writer, "The --exclusive option must be enabled for storage %s\n", + preference.StorageURL) + os.Exit(ArgumentExitCode) + } + + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil) + duplicacy.SavePassword(*preference, "password", password) + + backupManager.SetupSnapshotCache(preference.Name) + backupManager.SnapshotManager.PruneSnapshots(selfID, snapshotID, revisions, tags, retentions, + exhaustive, exclusive, ignoredIDs, dryRun, deleteOnly, collectOnly, threads) + + runScript(context, preference.Name, "post") +} + +func copySnapshots(context *cli.Context) { + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + if len(context.Args()) != 0 { + fmt.Fprintf(context.App.Writer, "The %s command requires no arguments.\n\n", context.Command.Name) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + uploadingThreads := context.Int("threads") + if uploadingThreads < 1 { + uploadingThreads = 1 + } + + downloadingThreads := context.Int("download-threads") + if downloadingThreads < 1 { + downloadingThreads = 1 + } + + repository, source := getRepositoryPreference(context, context.String("from")) + + runScript(context, source.Name, "pre") + + duplicacy.LOG_INFO("STORAGE_SET", "Source storage set to %s", source.StorageURL) + sourceStorage := duplicacy.CreateStorage(*source, false, downloadingThreads) + if sourceStorage == nil { + return + } + + sourcePassword := "" + if source.Encrypted { + sourcePassword = duplicacy.GetPassword(*source, "password", "Enter source storage password:", false, false) + } + + sourceManager := duplicacy.CreateBackupManager(source.SnapshotID, sourceStorage, repository, sourcePassword, nil) + sourceManager.SetupSnapshotCache(source.Name) + duplicacy.SavePassword(*source, "password", sourcePassword) + + loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), source, sourceManager, false) + + _, destination := getRepositoryPreference(context, context.String("to")) + + if destination.Name == source.Name { + duplicacy.LOG_ERROR("COPY_IDENTICAL", "The source storage and the destination storage are the same") + return + } + + if destination.BackupProhibited { + duplicacy.LOG_ERROR("COPY_DISABLED", "Copying snapshots to %s was disabled by the preference", + destination.StorageURL) + return + } + + duplicacy.LOG_INFO("STORAGE_SET", "Destination storage set to %s", destination.StorageURL) + destinationStorage := duplicacy.CreateStorage(*destination, false, uploadingThreads) + if destinationStorage == nil { + return + } + + destinationPassword := "" + if destination.Encrypted { + destinationPassword = duplicacy.GetPassword(*destination, "password", + "Enter destination storage password:", false, false) + } + + sourceStorage.SetRateLimits(context.Int("download-limit-rate"), 0) + destinationStorage.SetRateLimits(0, context.Int("upload-limit-rate")) + + destinationManager := duplicacy.CreateBackupManager(destination.SnapshotID, destinationStorage, repository, + destinationPassword, nil) + duplicacy.SavePassword(*destination, "password", destinationPassword) + destinationManager.SetupSnapshotCache(destination.Name) + + revisions := getRevisions(context) + snapshotID := "" + if context.String("id") != "" { + snapshotID = context.String("id") + } + + sourceManager.CopySnapshots(destinationManager, snapshotID, revisions, uploadingThreads, downloadingThreads) + runScript(context, source.Name, "post") +} + +func infoStorage(context *cli.Context) { + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + if len(context.Args()) != 1 { + fmt.Fprintf(context.App.Writer, "The %s command requires a storage URL argument.\n\n", context.Command.Name) + cli.ShowCommandHelp(context, context.Command.Name) + os.Exit(ArgumentExitCode) + } + + repository := context.String("repository") + if repository != "" { + preferencePath := path.Join(repository, duplicacy.DUPLICACY_DIRECTORY) + duplicacy.SetDuplicacyPreferencePath(preferencePath) + duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring")) + } + + resetPasswords := context.Bool("reset-passwords") + isEncrypted := context.Bool("e") + preference := duplicacy.Preference{ + Name: "default", + SnapshotID: "default", + StorageURL: context.Args()[0], + Encrypted: isEncrypted, + DoNotSavePassword: true, + } + + storageName := context.String("storage-name") + if storageName != "" { + preference.Name = storageName + } + + if resetPasswords { + // We don't want password entered for the info command to overwrite the saved password for the default storage, + // so we simply assign an empty name. + preference.Name = "" + } + + password := "" + if isEncrypted { + password = duplicacy.GetPassword(preference, "password", "Enter the storage password:", false, resetPasswords) + } + + storage := duplicacy.CreateStorage(preference, resetPasswords, 1) + config, isStorageEncrypted, err := duplicacy.DownloadConfig(storage, password) + + if isStorageEncrypted { + duplicacy.LOG_INFO("STORAGE_ENCRYPTED", "The storage is encrypted with a password") + } else if err != nil { + duplicacy.LOG_ERROR("STORAGE_ERROR", "%v", err) + } else if config == nil { + duplicacy.LOG_INFO("STORAGE_NOT_INITIALIZED", "The storage has not been initialized") + } else { + config.Print() + } + + dirs, _, err := storage.ListFiles(0, "snapshots/") + if err != nil { + duplicacy.LOG_WARN("STORAGE_LIST", "Failed to list repository ids: %v", err) + return + } + + for _, dir := range dirs { + if len(dir) > 0 && dir[len(dir)-1] == '/' { + duplicacy.LOG_INFO("STORAGE_SNAPSHOT", "%s", dir[0:len(dir)-1]) + } + } + +} + +func benchmark(context *cli.Context) { + setGlobalOptions(context) + defer duplicacy.CatchLogException() + + fileSize := context.Int("file-size") + if fileSize == 0 { + fileSize = 256 + } + + chunkCount := context.Int("chunk-count") + if chunkCount == 0 { + chunkCount = 64 + } + + chunkSize := context.Int("chunk-size") + if chunkSize == 0 { + chunkSize = 4 + } + + downloadThreads := context.Int("download-threads") + if downloadThreads < 1 { + downloadThreads = 1 + } + + uploadThreads := context.Int("upload-threads") + if uploadThreads < 1 { + uploadThreads = 1 + } + + threads := downloadThreads + if threads < uploadThreads { + threads = uploadThreads + } + + repository, preference := getRepositoryPreference(context, context.String("storage")) + + duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL) + storage := duplicacy.CreateStorage(*preference, false, threads) + if storage == nil { + return + } + duplicacy.Benchmark(repository, storage, int64(fileSize)*1024*1024, chunkSize*1024*1024, chunkCount, uploadThreads, downloadThreads) +} + +func main() { + + duplicacy.SetLoggingLevel(duplicacy.INFO) + + app := cli.NewApp() + + app.Commands = []cli.Command{ + { + Name: "init", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "encrypt, e", + Usage: "encrypt the storage with a password", + }, + cli.StringFlag{ + Name: "chunk-size, c", + Value: "4M", + Usage: "the average size of chunks (defaults to 4M)", + Argument: "", + }, + cli.StringFlag{ + Name: "max-chunk-size, max", + Usage: "the maximum size of chunks (defaults to chunk-size*4)", + Argument: "", + }, + cli.StringFlag{ + Name: "min-chunk-size, min", + Usage: "the minimum size of chunks (defaults to chunk-size/4)", + Argument: "", + }, + cli.StringFlag{ + Name: "zstd-level", + Usage: "set zstd compression level (fast, default, better, or best)", + Argument: "", + }, + cli.BoolFlag{ + Name: "zstd", + Usage: "short for -zstd default", + }, + cli.IntFlag{ + Name: "iterations", + Usage: "the number of iterations used in storage key derivation (default is 16384)", + Argument: "", + }, + cli.StringFlag{ + Name: "pref-dir", + Usage: "alternate location for the .duplicacy directory (absolute or relative to current directory)", + Argument: "", + }, + cli.StringFlag{ + Name: "storage-name", + Usage: "assign a name to the storage", + Argument: "", + }, + cli.StringFlag{ + Name: "repository", + Usage: "initialize a new repository at the specified path rather than the current working directory", + Argument: "", + }, + cli.StringFlag{ + Name: "key", + Usage: "the RSA public key to encrypt file chunks", + Argument: "", + }, + cli.StringFlag{ + Name: "erasure-coding", + Usage: "enable erasure coding to protect against storage corruption", + Argument: ":", + }, + }, + Usage: "Initialize the storage if necessary and the current directory as the repository", + ArgsUsage: " ", + Action: initRepository, + }, + { + Name: "backup", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "hash", + Usage: "detect file differences by hash (rather than size and timestamp)", + }, + cli.StringFlag{ + Name: "t", + Usage: "assign a tag to the backup", + Argument: "", + }, + cli.BoolFlag{ + Name: "stats", + Usage: "show statistics during and after backup", + }, + cli.IntFlag{ + Name: "threads", + Value: 1, + Usage: "number of uploading threads", + Argument: "", + }, + cli.IntFlag{ + Name: "limit-rate", + Value: 0, + Usage: "the maximum upload rate (in kilobytes/sec)", + Argument: "", + }, + cli.BoolFlag{ + Name: "dry-run", + Usage: "dry run for testing, don't backup anything. Use with -stats and -d", + }, + cli.StringFlag{ + Name: "zstd-level", + Usage: "set zstd compression level (fast, default, better, or best)", + Argument: "", + }, + cli.BoolFlag{ + Name: "zstd", + Usage: "short for -zstd default", + }, + cli.BoolFlag{ + Name: "vss", + Usage: "enable the Volume Shadow Copy service (Windows and macOS using APFS only)", + }, + cli.IntFlag{ + Name: "vss-timeout", + Value: 0, + Usage: "the timeout in seconds to wait for the Volume Shadow Copy operation to complete", + Argument: "", + }, + cli.StringFlag{ + Name: "storage", + Usage: "backup to the specified storage instead of the default one", + Argument: "", + }, + cli.BoolFlag{ + Name: "enum-only", + Usage: "enumerate the repository recursively and then exit", + }, + cli.IntFlag{ + Name: "metadata-chunk-size", + Value: 1024 * 1024, + Usage: "the average size of metadata chunks (defaults to 1M)", + Argument: "", + }, + cli.IntFlag{ + Name: "max-in-memory-entries", + Value: 1024 * 1024, + Usage: "the maximum number of entries kept in memory (defaults to 1M)", + Argument: "", + }, + }, + Usage: "Save a snapshot of the repository to the storage", + ArgsUsage: " ", + Action: backupRepository, + }, + + { + Name: "restore", + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "r", + Usage: "the revision number of the snapshot (required)", + Argument: "", + }, + cli.BoolFlag{ + Name: "hash", + Usage: "detect file differences by hash (rather than size and timestamp)", + }, + cli.BoolFlag{ + Name: "overwrite", + Usage: "overwrite existing files in the repository", + }, + cli.BoolFlag{ + Name: "delete", + Usage: "delete files not in the snapshot", + }, + cli.BoolFlag{ + Name: "ignore-owner", + Usage: "do not set the original uid/gid on restored files", + }, + cli.BoolFlag{ + Name: "stats", + Usage: "show statistics during and after restore", + }, + cli.IntFlag{ + Name: "threads", + Value: 1, + Usage: "number of downloading threads", + Argument: "", + }, + cli.IntFlag{ + Name: "limit-rate", + Value: 0, + Usage: "the maximum download rate (in kilobytes/sec)", + Argument: "", + }, + cli.StringFlag{ + Name: "storage", + Usage: "restore from the specified storage instead of the default one", + Argument: "", + }, + cli.StringFlag{ + Name: "key", + Usage: "the RSA private key to decrypt file chunks", + Argument: "", + }, + cli.BoolFlag{ + Name: "persist", + Usage: "continue processing despite chunk errors or existing files (without -overwrite), reporting any affected files", + }, + cli.StringFlag{ + Name: "key-passphrase", + Usage: "the passphrase to decrypt the RSA private key", + Argument: "", + }, + }, + Usage: "Restore the repository to a previously saved snapshot", + ArgsUsage: "[--] [pattern] ...", + Action: restoreRepository, + }, + + { + Name: "list", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "all, a", + Usage: "list snapshots with any id", + }, + cli.StringFlag{ + Name: "id", + Usage: "list snapshots with the specified id rather than the default one", + Argument: "", + }, + cli.StringSliceFlag{ + Name: "r", + Usage: "the revision number of the snapshot", + Argument: "", + }, + cli.StringFlag{ + Name: "t", + Usage: "list snapshots with the specified tag", + Argument: "", + }, + cli.BoolFlag{ + Name: "files", + Usage: "print the file list in each snapshot", + }, + cli.BoolFlag{ + Name: "chunks", + Usage: "print chunks in each snapshot or all chunks if no snapshot specified", + }, + cli.BoolFlag{ + Name: "reset-passwords", + Usage: "take passwords from input rather than keychain/keyring", + }, + cli.StringFlag{ + Name: "storage", + Usage: "retrieve snapshots from the specified storage", + Argument: "", + }, + cli.StringFlag{ + Name: "key", + Usage: "the RSA private key to decrypt file chunks", + Argument: "", + }, + }, + Usage: "List snapshots", + ArgsUsage: " ", + Action: listSnapshots, + }, + { + Name: "check", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "all, a", + Usage: "check snapshots with any id", + }, + cli.StringFlag{ + Name: "id", + Usage: "check snapshots with the specified id rather than the default one", + Argument: "", + }, + cli.StringSliceFlag{ + Name: "r", + Usage: "the revision number of the snapshot", + Argument: "", + }, + cli.StringFlag{ + Name: "t", + Usage: "check snapshots with the specified tag", + Argument: "", + }, + cli.BoolFlag{ + Name: "fossils", + Usage: "search fossils if a chunk can't be found", + }, + cli.BoolFlag{ + Name: "resurrect", + Usage: "turn referenced fossils back into chunks", + }, + cli.BoolFlag{ + Name: "rewrite", + Usage: "rewrite chunks with recoverable corruption", + }, + cli.BoolFlag{ + Name: "files", + Usage: "verify the integrity of every file", + }, + cli.BoolFlag{ + Name: "chunks", + Usage: "verify the integrity of every chunk", + }, + cli.BoolFlag{ + Name: "stats", + Usage: "show deduplication statistics (imply -all and all revisions)", + }, + cli.BoolFlag{ + Name: "tabular", + Usage: "show tabular usage and deduplication statistics (imply -stats, -all, and all revisions)", + }, + cli.StringFlag{ + Name: "storage", + Usage: "retrieve snapshots from the specified storage", + Argument: "", + }, + cli.StringFlag{ + Name: "key", + Usage: "the RSA private key to decrypt file chunks", + Argument: "", + }, + cli.StringFlag{ + Name: "key-passphrase", + Usage: "the passphrase to decrypt the RSA private key", + Argument: "", + }, + cli.IntFlag{ + Name: "threads", + Value: 1, + Usage: "number of threads used to verify chunks", + Argument: "", + }, + cli.BoolFlag{ + Name: "persist", + Usage: "continue processing despite chunk errors, reporting any affected (corrupted) files", + }, + }, + Usage: "Check the integrity of snapshots", + ArgsUsage: " ", + Action: checkSnapshots, + }, + { + Name: "cat", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "id", + Usage: "retrieve from the snapshot with the specified id", + Argument: "", + }, + cli.IntFlag{ + Name: "r", + Usage: "the revision number of the snapshot", + Argument: "", + }, + cli.StringFlag{ + Name: "storage", + Usage: "retrieve the file from the specified storage", + Argument: "", + }, + cli.StringFlag{ + Name: "key", + Usage: "the RSA private key to decrypt file chunks", + Argument: "", + }, + cli.StringFlag{ + Name: "key-passphrase", + Usage: "the passphrase to decrypt the RSA private key", + Argument: "", + }, + }, + Usage: "Print to stdout the specified file, or the snapshot content if no file is specified", + ArgsUsage: "[]", + Action: printFile, + }, + + { + Name: "diff", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "id", + Usage: "diff snapshots with the specified id", + Argument: "", + }, + cli.IntSliceFlag{ + Name: "r", + Usage: "the revision number of the snapshot", + Argument: "", + }, + cli.BoolFlag{ + Name: "hash", + Usage: "compute the hashes of on-disk files", + }, + cli.StringFlag{ + Name: "storage", + Usage: "retrieve files from the specified storage", + Argument: "", + }, + cli.StringFlag{ + Name: "key", + Usage: "the RSA private key to decrypt file chunks", + Argument: "", + }, + cli.StringFlag{ + Name: "key-passphrase", + Usage: "the passphrase to decrypt the RSA private key", + Argument: "", + }, + }, + Usage: "Compare two snapshots or two revisions of a file", + ArgsUsage: "[]", + Action: diff, + }, + + { + Name: "history", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "id", + Usage: "find the file in the snapshot with the specified id", + Argument: "", + }, + cli.StringSliceFlag{ + Name: "r", + Usage: "show history of the specified revisions", + Argument: "", + }, + cli.BoolFlag{ + Name: "hash", + Usage: "show the hash of the on-disk file", + }, + cli.StringFlag{ + Name: "storage", + Usage: "retrieve files from the specified storage", + Argument: "", + }, + }, + Usage: "Show the history of a file", + ArgsUsage: "", + Action: showHistory, + }, + + { + Name: "prune", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "id", + Usage: "delete snapshots with the specified id instead of the default one", + Argument: "", + }, + cli.BoolFlag{ + Name: "all, a", + Usage: "match against all snapshot IDs", + }, + cli.StringSliceFlag{ + Name: "r", + Usage: "delete snapshots with the specified revisions", + Argument: "", + }, + cli.StringSliceFlag{ + Name: "t", + Usage: "delete snapshots with the specified tags", + Argument: "", + }, + cli.StringSliceFlag{ + Name: "keep", + Usage: "keep 1 snapshot every n days for snapshots older than m days", + Argument: "", + }, + cli.BoolFlag{ + Name: "exhaustive", + Usage: "remove all unreferenced chunks (not just those referenced by deleted snapshots)", + }, + cli.BoolFlag{ + Name: "exclusive", + Usage: "assume exclusive access to the storage (disable two-step fossil collection)", + }, + cli.BoolFlag{ + Name: "dry-run, d", + Usage: "show what would have been deleted", + }, + cli.BoolFlag{ + Name: "delete-only", + Usage: "delete fossils previously collected (if deletable) and don't collect fossils", + }, + cli.BoolFlag{ + Name: "collect-only", + Usage: "identify and collect fossils, but don't delete fossils previously collected", + }, + cli.StringSliceFlag{ + Name: "ignore", + Usage: "ignore snapshots with the specified id when deciding if fossils can be deleted", + Argument: "", + }, + cli.StringFlag{ + Name: "storage", + Usage: "prune snapshots from the specified storage", + Argument: "", + }, + cli.IntFlag{ + Name: "threads", + Value: 1, + Usage: "number of threads used to prune unreferenced chunks", + Argument: "", + }, + }, + Usage: "Prune snapshots by revision, tag, or retention policy", + ArgsUsage: " ", + Action: pruneSnapshots, + }, + + { + Name: "password", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "storage", + Usage: "change the password used to access the specified storage", + Argument: "", + }, + cli.IntFlag{ + Name: "iterations", + Usage: "the number of iterations used in storage key derivation (default is 16384)", + Argument: "", + }, + }, + Usage: "Change the storage password", + ArgsUsage: " ", + Action: changePassword, + }, + + { + Name: "add", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "encrypt, e", + Usage: "encrypt the storage with a password", + }, + cli.StringFlag{ + Name: "chunk-size, c", + Value: "4M", + Usage: "the average size of chunks (default is 4M)", + Argument: "", + }, + cli.StringFlag{ + Name: "max-chunk-size, max", + Usage: "the maximum size of chunks (default is chunk-size*4)", + Argument: "", + }, + cli.StringFlag{ + Name: "min-chunk-size, min", + Usage: "the minimum size of chunks (default is chunk-size/4)", + Argument: "", + }, + cli.StringFlag{ + Name: "zstd-level", + Usage: "set zstd compression level (fast, default, better, or best)", + Argument: "", + }, + cli.BoolFlag{ + Name: "zstd", + Usage: "short for -zstd default", + }, + cli.IntFlag{ + Name: "iterations", + Usage: "the number of iterations used in storage key derivation (default is 16384)", + Argument: "", + }, + cli.StringFlag{ + Name: "copy", + Usage: "make the new storage compatible with an existing one to allow for copy operations", + Argument: "", + }, + cli.BoolFlag{ + Name: "bit-identical", + Usage: "(when using -copy) make the new storage bit-identical to also allow rsync etc.", + }, + cli.StringFlag{ + Name: "repository", + Usage: "specify the path of the repository (instead of the current working directory)", + Argument: "", + }, + cli.StringFlag{ + Name: "key", + Usage: "the RSA public key to encrypt file chunks", + Argument: "", + }, + cli.StringFlag{ + Name: "erasure-coding", + Usage: "enable erasure coding to protect against storage corruption", + Argument: ":", + }, + }, + Usage: "Add an additional storage to be used for the existing repository", + ArgsUsage: " ", + Action: addStorage, + }, + + { + Name: "set", + Flags: []cli.Flag{ + cli.GenericFlag{ + Name: "encrypt, e", + Usage: "encrypt the storage with a password", + Value: &TriBool{}, + Arg: "true", + }, + cli.GenericFlag{ + Name: "no-backup", + Usage: "backup to this storage is prohibited", + Value: &TriBool{}, + Arg: "true", + }, + cli.GenericFlag{ + Name: "no-restore", + Usage: "restore from this storage is prohibited", + Value: &TriBool{}, + Arg: "true", + }, + cli.GenericFlag{ + Name: "no-save-password", + Usage: "don't save password or access keys to keychain/keyring", + Value: &TriBool{}, + Arg: "true", + }, + cli.StringFlag{ + Name: "nobackup-file", + Usage: "Directories containing a file with this name will not be backed up", + Argument: "", + Value: "", + }, + cli.GenericFlag{ + Name: "exclude-by-attribute", + Usage: "Exclude files based on file attributes. (macOS only, com_apple_backup_excludeItem)", + Value: &TriBool{}, + Arg: "true", + }, + cli.StringFlag{ + Name: "key", + Usage: "add a key/password whose value is supplied by the -value option", + }, + cli.StringFlag{ + Name: "value", + Usage: "the value of the key/password", + }, + cli.StringFlag{ + Name: "storage", + Usage: "use the specified storage instead of the default one", + Argument: "", + }, + cli.StringFlag{ + Name: "filters", + Usage: "specify the path of the filters file containing include/exclude patterns", + Argument: "", + }, + }, + Usage: "Change the options for the default or specified storage", + ArgsUsage: " ", + Action: setPreference, + }, + { + Name: "copy", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "id", + Usage: "copy snapshots with the specified id instead of all snapshot ids", + Argument: "", + }, + cli.StringSliceFlag{ + Name: "r", + Usage: "copy snapshots with the specified revisions", + Argument: "", + }, + cli.StringFlag{ + Name: "from", + Usage: "copy snapshots from the specified storage", + Argument: "", + }, + cli.StringFlag{ + Name: "to", + Usage: "copy snapshots to the specified storage", + Argument: "", + }, + cli.IntFlag{ + Name: "download-limit-rate", + Value: 0, + Usage: "the maximum download rate (in kilobytes/sec)", + Argument: "", + }, + cli.IntFlag{ + Name: "upload-limit-rate", + Value: 0, + Usage: "the maximum upload rate (in kilobytes/sec)", + Argument: "", + }, + cli.IntFlag{ + Name: "threads", + Value: 1, + Usage: "number of uploading threads", + Argument: "", + }, + cli.IntFlag{ + Name: "download-threads", + Value: 1, + Usage: "number of downloading threads", + Argument: "", + }, + cli.StringFlag{ + Name: "key", + Usage: "the RSA private key to decrypt file chunks from the source storage", + Argument: "", + }, + cli.StringFlag{ + Name: "key-passphrase", + Usage: "the passphrase to decrypt the RSA private key", + Argument: "", + }, + }, + Usage: "Copy snapshots between compatible storages", + ArgsUsage: " ", + Action: copySnapshots, + }, + + { + Name: "info", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "encrypt, e", + Usage: "The storage is encrypted with a password", + }, + cli.StringFlag{ + Name: "repository", + Usage: "retrieve saved passwords from the specified repository", + Argument: "", + }, + cli.StringFlag{ + Name: "storage-name", + Usage: "the storage name to be assigned to the storage url", + Argument: "", + }, + cli.BoolFlag{ + Name: "reset-passwords", + Usage: "take passwords from input rather than keychain/keyring", + }, + }, + Usage: "Show the information about the specified storage", + ArgsUsage: "", + Action: infoStorage, + }, + + { + Name: "benchmark", + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "file-size", + Usage: "the size of the local file to write to and read from (in MB, default to 256)", + Argument: "", + }, + cli.IntFlag{ + Name: "chunk-count", + Usage: "the number of chunks to upload and download (default to 64)", + Argument: "", + }, + cli.IntFlag{ + Name: "chunk-size", + Usage: "the size of chunks to upload and download (in MB, default to 4)", + Argument: "", + }, + cli.IntFlag{ + Name: "upload-threads", + Usage: "the number of upload threads (default to 1)", + Argument: "", + }, + cli.IntFlag{ + Name: "download-threads", + Usage: "the number of download threads (default to 1)", + Argument: "", + }, + cli.StringFlag{ + Name: "storage", + Usage: "run the download/upload test agaist the specified storage", + Argument: "", + }, + }, + Usage: "Run a set of benchmarks to test download and upload speeds", + ArgsUsage: " ", + Action: benchmark, + }, + } + + app.Flags = []cli.Flag{ + cli.BoolFlag{ + Name: "verbose, v", + Usage: "show more detailed information", + }, + cli.BoolFlag{ + Name: "debug, d", + Usage: "show even more detailed information, useful for debugging", + }, + cli.BoolFlag{ + Name: "log", + Usage: "enable log-style output", + }, + cli.BoolFlag{ + Name: "stack", + Usage: "print the stack trace when an error occurs", + }, + cli.BoolFlag{ + Name: "no-script", + Usage: "do not run script before or after command execution", + }, + cli.BoolFlag{ + Name: "background", + Usage: "read passwords, tokens, or keys only from keychain/keyring or env", + }, + cli.StringFlag{ + Name: "profile", + Value: "", + Usage: "enable the profiling tool and listen on the specified address:port", + Argument: "", + }, + cli.StringFlag{ + Name: "comment", + Usage: "add a comment to identify the process", + }, + cli.StringSliceFlag{ + Name: "suppress, s", + Usage: "suppress logs with the specified id", + Argument: "", + }, + cli.BoolFlag{ + Name: "print-memory-usage", + Usage: "print memory usage every second", + }, + } + + app.HideVersion = true + app.Name = "dupluxy" + app.HelpName = "dupluxy" + app.Usage = "Duplicacy derived cloud backup tool" + app.Version = Version + " (" + GitCommit + ")" + + // Exit with code 2 if an invalid command is provided + app.CommandNotFound = func(context *cli.Context, command string) { + fmt.Fprintf(context.App.Writer, "Invalid command: %s\n", command) + os.Exit(2) + } + + // If the program is interrupted, call the RunAtError function. + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + go func() { + for range c { + duplicacy.RunAtError() + os.Exit(1) + } + }() + + err := app.Run(os.Args) + if err != nil { + os.Exit(2) + } + +} diff --git a/go.mod b/go.mod index 7b82dfd..8e24cd7 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/gilbertchen/duplicacy +module github.com/dupluxy/dupluxy go 1.20