diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index 65943c9..5e29d0e 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -583,6 +583,11 @@ func setPreference(context *cli.Context) { newPreference.FiltersFile = context.String("filters") } + triBool = context.Generic("exclude-by-attribute").(*TriBool) + if triBool.IsSet() { + newPreference.ExcludeByAttribute = triBool.IsTrue() + } + key := context.String("key") value := context.String("value") @@ -764,7 +769,7 @@ func backupRepository(context *cli.Context) { uploadRateLimit := context.Int("limit-rate") enumOnly := context.Bool("enum-only") storage.SetRateLimits(0, uploadRateLimit) - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile) + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile, preference.ExcludeByAttribute) duplicacy.SavePassword(*preference, "password", password) backupManager.SetupSnapshotCache(preference.Name) @@ -842,7 +847,7 @@ func restoreRepository(context *cli.Context) { duplicacy.LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) storage.SetRateLimits(context.Int("limit-rate"), 0) - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile) + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile, preference.ExcludeByAttribute) duplicacy.SavePassword(*preference, "password", password) loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false) @@ -888,7 +893,7 @@ func listSnapshots(context *cli.Context) { tag := context.String("t") revisions := getRevisions(context) - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "") + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", preference.ExcludeByAttribute) duplicacy.SavePassword(*preference, "password", password) id := preference.SnapshotID @@ -944,7 +949,7 @@ func checkSnapshots(context *cli.Context) { tag := context.String("t") revisions := getRevisions(context) - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "") + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false) duplicacy.SavePassword(*preference, "password", password) loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false) @@ -1003,7 +1008,8 @@ func printFile(context *cli.Context) { snapshotID = context.String("id") } - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "") + + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false) duplicacy.SavePassword(*preference, "password", password) loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false) @@ -1061,13 +1067,13 @@ func diff(context *cli.Context) { } compareByHash := context.Bool("hash") - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "") + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false) duplicacy.SavePassword(*preference, "password", password) loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false) backupManager.SetupSnapshotCache(preference.Name) - backupManager.SnapshotManager.Diff(repository, snapshotID, revisions, path, compareByHash, preference.NobackupFile, preference.FiltersFile) + backupManager.SnapshotManager.Diff(repository, snapshotID, revisions, path, compareByHash, preference.NobackupFile, preference.FiltersFile, preference.ExcludeByAttribute) runScript(context, preference.Name, "post") } @@ -1106,7 +1112,7 @@ func showHistory(context *cli.Context) { revisions := getRevisions(context) showLocalHash := context.Bool("hash") - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "") + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false) duplicacy.SavePassword(*preference, "password", password) backupManager.SetupSnapshotCache(preference.Name) @@ -1169,7 +1175,7 @@ func pruneSnapshots(context *cli.Context) { os.Exit(ArgumentExitCode) } - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "") + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false) duplicacy.SavePassword(*preference, "password", password) backupManager.SetupSnapshotCache(preference.Name) @@ -1214,7 +1220,7 @@ func copySnapshots(context *cli.Context) { sourcePassword = duplicacy.GetPassword(*source, "password", "Enter source storage password:", false, false) } - sourceManager := duplicacy.CreateBackupManager(source.SnapshotID, sourceStorage, repository, sourcePassword, "", "") + sourceManager := duplicacy.CreateBackupManager(source.SnapshotID, sourceStorage, repository, sourcePassword, "", "", false) sourceManager.SetupSnapshotCache(source.Name) duplicacy.SavePassword(*source, "password", sourcePassword) @@ -1249,7 +1255,7 @@ func copySnapshots(context *cli.Context) { destinationStorage.SetRateLimits(0, context.Int("upload-limit-rate")) destinationManager := duplicacy.CreateBackupManager(destination.SnapshotID, destinationStorage, repository, - destinationPassword, "", "") + destinationPassword, "", "", fasle) duplicacy.SavePassword(*destination, "password", destinationPassword) destinationManager.SetupSnapshotCache(destination.Name) @@ -1968,6 +1974,12 @@ func main() { Argument: "", Value: "", }, + cli.GenericFlag{ + Name: "exclude-by-attribute", + Usage: "Exclude files based on file attributes. (macOS only, com_apple_backup_excludeItem)", + Value: &TriBool{}, + Arg: "true", + }, cli.StringFlag{ Name: "key", Usage: "add a key/password whose value is supplied by the -value option", diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index e76573e..40e685c 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -35,7 +35,11 @@ type BackupManager struct { config *Config // contains a number of options nobackupFile string // don't backup directory when this file name is found - filtersFile string // the path to the filters file + + filtersFile string // the path to the filters file + + excludeByAttribute bool // don't backup file based on file attribute + } func (manager *BackupManager) SetDryRun(dryRun bool) { @@ -45,7 +49,7 @@ func (manager *BackupManager) SetDryRun(dryRun bool) { // CreateBackupManager creates a backup manager using the specified 'storage'. 'snapshotID' is a unique id to // identify snapshots created for this repository. 'top' is the top directory of the repository. 'password' is the // master key which can be nil if encryption is not enabled. -func CreateBackupManager(snapshotID string, storage Storage, top string, password string, nobackupFile string, filtersFile string) *BackupManager { +func CreateBackupManager(snapshotID string, storage Storage, top string, password string, nobackupFile string, filtersFile string, excludeByAttribute bool) *BackupManager { config, _, err := DownloadConfig(storage, password) if err != nil { @@ -68,7 +72,10 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor config: config, nobackupFile: nobackupFile, - filtersFile: filtersFile, + + filtersFile: filtersFile, + + excludeByAttribute: excludeByAttribute, } if IsDebugging() { @@ -206,7 +213,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta LOG_INFO("BACKUP_INDEXING", "Indexing %s", top) localSnapshot, skippedDirectories, skippedFiles, err := CreateSnapshotFromDirectory(manager.snapshotID, shadowTop, - manager.nobackupFile, manager.filtersFile) + manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list the directory %s: %v", top, err) return false @@ -789,7 +796,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu manager.SnapshotManager.DownloadSnapshotContents(remoteSnapshot, patterns, true) localSnapshot, _, _, err := CreateSnapshotFromDirectory(manager.snapshotID, top, manager.nobackupFile, - manager.filtersFile) + manager.filtersFile, manager.excludeByAttribute) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list the repository: %v", err) return 0 diff --git a/src/duplicacy_backupmanager_test.go b/src/duplicacy_backupmanager_test.go index e2e64e1..15aeaab 100644 --- a/src/duplicacy_backupmanager_test.go +++ b/src/duplicacy_backupmanager_test.go @@ -253,7 +253,7 @@ func TestBackupManager(t *testing.T) { time.Sleep(time.Duration(delay) * time.Second) SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") - backupManager := CreateBackupManager("host1", storage, testDir, password, "", "") + backupManager := CreateBackupManager("host1", storage, testDir, password, "", "", false) backupManager.SetupSnapshotCache("default") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") diff --git a/src/duplicacy_entry.go b/src/duplicacy_entry.go index aae7c2d..0069904 100644 --- a/src/duplicacy_entry.go +++ b/src/duplicacy_entry.go @@ -28,6 +28,29 @@ var fileModeMask = os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky // Regex for matching 'StartChunk:StartOffset:EndChunk:EndOffset' var contentRegex = regexp.MustCompile(`^([0-9]+):([0-9]+):([0-9]+):([0-9]+)`) +// AttributeExcludeName attribute name to determine file exclusion +var AttributeExcludeName = getDefaultAttributeExcludeName() + +// AttributeExcludeValue attribute value to determine file exclusion +var AttributeExcludeValue = getDefaultAttributeExcludeValue() + +func getDefaultAttributeExcludeName() string { + if runtime.GOOS == "darwin" { + return "com.apple.metadata:com_apple_backup_excludeItem" + } + if runtime.GOOS == "linux" { + return "duplicacy_exclude" + } + return "" +} + +func getDefaultAttributeExcludeValue() string { + if runtime.GOOS == "darwin" { + return "com.apple.backupd" + } + return "" +} + // Entry encapsulates information about a file or directory. type Entry struct { Path string @@ -443,7 +466,7 @@ func (files FileInfoCompare) Less(i, j int) bool { // ListEntries returns a list of entries representing file and subdirectories under the directory 'path'. Entry paths // are normalized as relative to 'top'. 'patterns' are used to exclude or include certain files. -func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, nobackupFile string, discardAttributes bool) (directoryList []*Entry, +func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, nobackupFile string, discardAttributes bool, excludeByAttribute bool) (directoryList []*Entry, skippedFiles []string, err error) { LOG_DEBUG("LIST_ENTRIES", "Listing %s", path) @@ -524,6 +547,17 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, entry.ReadAttributes(top) } + if excludeByAttribute && (runtime.GOOS == "darwin" || runtime.GOOS == "linux") { + attrValue, ok := entry.Attributes[AttributeExcludeName] + if ok { + attrValueString := string(attrValue) + if strings.Contains(attrValueString, AttributeExcludeValue) { + LOG_WARN("LIST_NOBACKUPXATTR", "%s is excluded due to extended attribute: %s", entry.Path, AttributeExcludeName) + continue + } + } + } + if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 { LOG_WARN("LIST_SKIP", "Skipped non-regular file %s", entry.Path) skippedFiles = append(skippedFiles, entry.Path) diff --git a/src/duplicacy_entry_test.go b/src/duplicacy_entry_test.go index d809412..246d17d 100644 --- a/src/duplicacy_entry_test.go +++ b/src/duplicacy_entry_test.go @@ -9,8 +9,12 @@ import ( "math/rand" "os" "path/filepath" + "runtime" "sort" + "strings" "testing" + + "github.com/gilbertchen/xattr" ) func TestEntrySort(t *testing.T) { @@ -173,7 +177,7 @@ func TestEntryList(t *testing.T) { directory := directories[len(directories)-1] directories = directories[:len(directories)-1] entries = append(entries, directory) - subdirectories, _, err := ListEntries(testDir, directory.Path, &entries, nil, "", false) + subdirectories, _, err := ListEntries(testDir, directory.Path, &entries, nil, "", false, false) if err != nil { t.Errorf("ListEntries(%s, %s) returned an error: %s", testDir, directory.Path, err) } @@ -216,3 +220,110 @@ func TestEntryList(t *testing.T) { } } + +// TestEntryExcludeByAttribute tests the excludeByAttribute parameter to the ListEntries function +func TestEntryExcludeByAttribute(t *testing.T) { + + if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") { + t.Skip("skipping test not darwin or linux") + } + + testDir := filepath.Join(os.TempDir(), "duplicacy_test") + + os.RemoveAll(testDir) + os.MkdirAll(testDir, 0700) + + // Files or folders named with "exclude" below will have the exclusion attribute set on them + // When ListEntries is called with excludeByAttribute true, they should be excluded. + DATA := [...]string{ + "excludefile", + "includefile", + "excludedir/", + "excludedir/file", + "includedir/", + "includedir/includefile", + "includedir/excludefile", + } + + for _, file := range DATA { + fullPath := filepath.Join(testDir, file) + if file[len(file)-1] == '/' { + err := os.Mkdir(fullPath, 0700) + if err != nil { + t.Errorf("Mkdir(%s) returned an error: %s", fullPath, err) + } + continue + } + + err := ioutil.WriteFile(fullPath, []byte(file), 0700) + if err != nil { + t.Errorf("WriteFile(%s) returned an error: %s", fullPath, err) + } + } + + for _, file := range DATA { + fullPath := filepath.Join(testDir, file) + if strings.Contains(file, "exclude") { + xattr.Setxattr(fullPath, AttributeExcludeName, []byte(AttributeExcludeValue)) + } + } + + for _, excludeByAttribute := range [2]bool{true, false} { + t.Logf("testing excludeByAttribute: %t", excludeByAttribute) + directories := make([]*Entry, 0, 4) + directories = append(directories, CreateEntry("", 0, 0, 0)) + + entries := make([]*Entry, 0, 4) + + for len(directories) > 0 { + directory := directories[len(directories)-1] + directories = directories[:len(directories)-1] + entries = append(entries, directory) + subdirectories, _, err := ListEntries(testDir, directory.Path, &entries, nil, "", false, excludeByAttribute) + if err != nil { + t.Errorf("ListEntries(%s, %s) returned an error: %s", testDir, directory.Path, err) + } + directories = append(directories, subdirectories...) + } + + entries = entries[1:] + + for _, entry := range entries { + t.Logf("entry: %s", entry.Path) + } + + i := 0 + for _, file := range DATA { + entryFound := false + var entry *Entry + for _, entry = range entries { + if entry.Path == file { + entryFound = true + break + } + } + + if excludeByAttribute && strings.Contains(file, "exclude") { + if entryFound { + t.Errorf("file: %s, expected to be excluded but wasn't. attributes: %v", file, entry.Attributes) + i++ + } else { + t.Logf("file: %s, excluded", file) + } + } else { + if entryFound { + t.Logf("file: %s, included. attributes: %v", file, entry.Attributes) + i++ + } else { + t.Errorf("file: %s, expected to be included but wasn't", file) + } + } + } + + } + + if !t.Failed() { + os.RemoveAll(testDir) + } + +} diff --git a/src/duplicacy_preference.go b/src/duplicacy_preference.go index 6ec3904..6a77a44 100644 --- a/src/duplicacy_preference.go +++ b/src/duplicacy_preference.go @@ -26,6 +26,7 @@ type Preference struct { NobackupFile string `json:"nobackup_file"` Keys map[string]string `json:"keys"` FiltersFile string `json:"filters"` + ExcludeByAttribute bool `json:"exclude_by_attribute"` } var preferencePath string diff --git a/src/duplicacy_snapshot.go b/src/duplicacy_snapshot.go index 45180f9..61be0cd 100644 --- a/src/duplicacy_snapshot.go +++ b/src/duplicacy_snapshot.go @@ -58,7 +58,7 @@ func CreateEmptySnapshot(id string) (snapshto *Snapshot) { // CreateSnapshotFromDirectory creates a snapshot from the local directory 'top'. Only 'Files' // will be constructed, while 'ChunkHashes' and 'ChunkLengths' can only be populated after uploading. -func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, filtersFile string) (snapshot *Snapshot, skippedDirectories []string, +func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, filtersFile string, , excludeByAttribute bool) (snapshot *Snapshot, skippedDirectories []string, skippedFiles []string, err error) { snapshot = &Snapshot{ @@ -89,7 +89,7 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, fil directory := directories[len(directories)-1] directories = directories[:len(directories)-1] snapshot.Files = append(snapshot.Files, directory) - subdirectories, skipped, err := ListEntries(top, directory.Path, &snapshot.Files, patterns, nobackupFile, snapshot.discardAttributes) + subdirectories, skipped, err := ListEntries(top, directory.Path, &snapshot.Files, patterns, nobackupFile, snapshot.discardAttributes, excludeByAttribute) if err != nil { if directory.Path == "" { LOG_ERROR("LIST_FAILURE", "Failed to list the repository root: %v", err) diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index dfa9e6d..50ffeaa 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -1414,7 +1414,7 @@ func (manager *SnapshotManager) PrintFile(snapshotID string, revision int, path // Diff compares two snapshots, or two revision of a file if the file argument is given. func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []int, - filePath string, compareByHash bool, nobackupFile string, filtersFile string) bool { + filePath string, compareByHash bool, nobackupFile string, filtersFile string, excludeByAttribute bool) bool { LOG_DEBUG("DIFF_PARAMETERS", "top: %s, id: %s, revision: %v, path: %s, compareByHash: %t", top, snapshotID, revisions, filePath, compareByHash) @@ -1427,7 +1427,7 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions [] if len(revisions) <= 1 { // Only scan the repository if filePath is not provided if len(filePath) == 0 { - rightSnapshot, _, _, err = CreateSnapshotFromDirectory(snapshotID, top, nobackupFile, filtersFile) + rightSnapshot, _, _, err = CreateSnapshotFromDirectory(snapshotID, top, nobackupFile, filtersFile, excludeByAttribute) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list the directory %s: %v", top, err) return false