diff --git a/src/duplicacy_entry.go b/src/duplicacy_entry.go index cf9db7e..c640865 100644 --- a/src/duplicacy_entry.go +++ b/src/duplicacy_entry.go @@ -481,9 +481,6 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, entries := make([]*Entry, 0, 4) for _, f := range files { - if (f.Name() == DUPLICACY_DIRECTORY) && (nobackupFile == "") { - continue - } entry := CreateEntryFromFileInfo(f, normalizedPath) if len(patterns) > 0 && !MatchPath(entry.Path, patterns) { continue diff --git a/src/duplicacy_snapshot.go b/src/duplicacy_snapshot.go index fd2e928..956c250 100644 --- a/src/duplicacy_snapshot.go +++ b/src/duplicacy_snapshot.go @@ -12,6 +12,7 @@ import ( "os" "path" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -69,18 +70,7 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn var patterns []string - patterns = ProcessFilterFile(path.Join(GetDuplicacyPreferencePath(), "filters"), make([]string, 0)) - - LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap)) - - LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) - - if IsTracing() { - for _, pattern := range patterns { - LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern) - } - - } + patterns = ProcessFilters() directories := make([]*Entry, 0, 256) directories = append(directories, CreateEntry("", 0, 0, 0)) @@ -122,6 +112,43 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn return snapshot, skippedDirectories, skippedFiles, nil } +func AppendPattern(patterns []string, new_pattern string) (new_patterns []string) { + for _, pattern := range patterns { + if pattern == new_pattern { + LOG_INFO("SNAPSHOT_FILTER", "Ignoring duplicate pattern: %s ...", new_pattern) + return patterns + } + } + new_patterns = append(patterns, new_pattern) + return new_patterns +} +func ProcessFilters() (patterns []string) { + patternFileLines := []string{ + `# ============================ exclude the internal files and directories in all ".duplicacy" subfolders`, + `e:(?i)(^|/)` + regexp.QuoteMeta(DUPLICACY_DIRECTORY) + `/cache/`, + `e:(?i)(^|/)` + regexp.QuoteMeta(DUPLICACY_DIRECTORY) + `/temporary$`, + `# ============================ exclude the internal files and directories in toplevel ".duplicacy" subfolder`, + `e:(?i)^` + regexp.QuoteMeta(DUPLICACY_DIRECTORY) + `/incomplete$`, + `e:(?i)^` + regexp.QuoteMeta(DUPLICACY_DIRECTORY) + `/logs/`, + "@" + joinPath(GetDuplicacyPreferencePath(), "filters"), + } + LOG_DEBUG("SNAPSHOT_FILTER", "Adding standard filters ...") + patterns = ProcessFilterLines(patternFileLines, make([]string, 0)) + + LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap)) + + LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) + + if IsTracing() { + for _, pattern := range patterns { + LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern) + } + + } + + return patterns +} + func ProcessFilterFile(patternFile string, includedFiles []string) (patterns []string) { if Contains(includedFiles, patternFile) { // cycle in include mechanism discovered. @@ -132,44 +159,59 @@ func ProcessFilterFile(patternFile string, includedFiles []string) (patterns []s LOG_INFO("SNAPSHOT_FILTER", "Parsing filter file %s ...", patternFile) patternFileContent, err := ioutil.ReadFile(patternFile) if err == nil { - for _, pattern := range strings.Split(string(patternFileContent), "\n") { - pattern = strings.TrimSpace(pattern) - if len(pattern) == 0 { - continue - } + patternFileLines := strings.Split(string(patternFileContent), "\n") + patterns = ProcessFilterLines(patternFileLines, includedFiles) + } + return patterns +} - if strings.HasPrefix(pattern, "@") { - patternIncludeFile := strings.TrimSpace(pattern[1:]) - if ! filepath.IsAbs(patternIncludeFile) { - patternIncludeFile = joinPath(filepath.Dir(patternFile), patternIncludeFile) - } - for _, pattern := range ProcessFilterFile(patternIncludeFile, includedFiles) { - patterns = append(patterns, pattern) - } - continue - } - - if pattern[0] == '#' { - continue - } - - if IsUnspecifiedFilter(pattern) { - pattern = "+" + pattern - } - - if IsEmptyFilter(pattern) { - continue - } - - if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") { - valid, err := IsValidRegex(pattern[2:]) - if !valid || err != nil { - LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err) - } - } - - patterns = append(patterns, pattern) +func ProcessFilterLines(patternFileLines []string, includedFiles []string) (patterns []string) { + for _, pattern := range patternFileLines { + pattern = strings.TrimSpace(pattern) + if len(pattern) == 0 { + continue } + + if strings.HasPrefix(pattern, "@") { + patternIncludeFile := strings.TrimSpace(pattern[1:]) + if patternIncludeFile == "" { + continue + } + if ! filepath.IsAbs(patternIncludeFile) { + basePath := "" + if len(includedFiles) == 0 { + basePath, _ = os.Getwd() + } else { + basePath = filepath.Dir(includedFiles[len(includedFiles)-1]) + } + patternIncludeFile = joinPath(basePath, patternIncludeFile) + } + for _, pattern := range ProcessFilterFile(patternIncludeFile, includedFiles) { + patterns = AppendPattern(patterns, pattern) + } + continue + } + + if pattern[0] == '#' { + continue + } + + if IsUnspecifiedFilter(pattern) { + pattern = "+" + pattern + } + + if IsEmptyFilter(pattern) { + continue + } + + if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") { + valid, err := IsValidRegex(pattern[2:]) + if !valid || err != nil { + LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err) + } + } + + patterns = AppendPattern(patterns, pattern) } return patterns