Add metadata and special files restore options

Add options to control if xattrs and specials get restored. Add
file flag mask preference. It's a rough interface, but this is a niche
use case.

Refactoring of options passing since golang ergonomics are poor for long
parameter lists.

duplicacy/duplicacy_main.go

@@ -858,6 +858,10 @@ func restoreRepository(context *cli.Context) {
 		SetOwner: !context.Bool("ignore-owner"),
 		ShowStatistics: context.Bool("stats"),
 		AllowFailures: context.Bool("persist"),
+		ExcludeXattrs: preference.ExcludeXattrs,
+		NormalizeXattrs: preference.NormalizeXattrs,
+		IncludeSpecials: preference.IncludeSpecials,
+		FileFlagsMask: uint32(preference.FileFlagsMask),
 	}
 
 	var patterns []string

src/duplicacy_backupmanager.go

@@ -45,15 +45,19 @@ type BackupOptions struct {
 }
 
 type RestoreOptions struct {
-	Threads        int
-	Patterns       []string
-	InPlace        bool
-	QuickMode      bool
-	Overwrite      bool
-	DeleteMode     bool
-	SetOwner       bool
-	ShowStatistics bool
-	AllowFailures  bool
+	Threads         int
+	Patterns        []string
+	InPlace         bool
+	QuickMode       bool
+	Overwrite       bool
+	DeleteMode      bool
+	SetOwner        bool
+	ShowStatistics  bool
+	AllowFailures   bool
+	ExcludeXattrs   bool
+	NormalizeXattrs bool
+	IncludeSpecials bool
+	FileFlagsMask   uint32
 }
 
 func (manager *BackupManager) SetDryRun(dryRun bool) {
@@ -648,6 +652,13 @@ func (manager *BackupManager) Restore(top string, revision int, options RestoreO
 	overwrite := options.Overwrite
 	allowFailures := options.AllowFailures
 
+	metadataOptions := RestoreMetadataOptions{
+		SetOwner: options.SetOwner,
+		ExcludeXattrs: options.ExcludeXattrs,
+		NormalizeXattrs: options.NormalizeXattrs,
+		FileFlagsMask: options.FileFlagsMask,
+	}
+
 	startTime := time.Now().Unix()
 
 	LOG_DEBUG("RESTORE_PARAMETERS", "top: %s, revision: %d, in-place: %t, quick: %t, delete: %t",
@@ -800,7 +811,7 @@ func (manager *BackupManager) Restore(top string, revision int, options RestoreO
 				if stat.Mode()&os.ModeSymlink != 0 {
 					isRegular, link, err := Readlink(fullPath)
 					if err == nil && link == remoteEntry.Link && !isRegular {
-						remoteEntry.RestoreMetadata(fullPath, nil, options.SetOwner)
+						remoteEntry.RestoreMetadata(fullPath, stat, metadataOptions)
 						if remoteEntry.IsHardLinkRoot() {
 							hardLinkTable[len(hardLinkTable)-1].willExist = true
 						}
@@ -825,7 +836,7 @@ func (manager *BackupManager) Restore(top string, revision int, options RestoreO
 				LOG_ERROR("RESTORE_SYMLINK", "Can't create symlink %s: %v", remoteEntry.Path, err)
 				return 0
 			}
-			remoteEntry.RestoreMetadata(fullPath, nil, options.SetOwner)
+			remoteEntry.RestoreMetadata(fullPath, nil, metadataOptions)
 			LOG_TRACE("DOWNLOAD_DONE", "Symlink %s updated", remoteEntry.Path)
 
 		} else if remoteEntry.IsDir() {
@@ -846,15 +857,15 @@ func (manager *BackupManager) Restore(top string, revision int, options RestoreO
 					return 0
 				}
 			}
-			err = remoteEntry.RestoreEarlyDirFlags(fullPath, 0) // TODO: mask
+			err = remoteEntry.RestoreEarlyDirFlags(fullPath, options.FileFlagsMask)
 			if err != nil {
 				LOG_WARN("DOWNLOAD_FLAGS", "Failed to set early file flags on %s: %v", fullPath, err)
 			}
 			directoryEntries = append(directoryEntries, remoteEntry)
-		} else if remoteEntry.IsSpecial() {
+		} else if remoteEntry.IsSpecial() && options.IncludeSpecials {
 			if stat, _ := os.Lstat(fullPath); stat != nil {
 				if remoteEntry.IsSameSpecial(stat) {
-					remoteEntry.RestoreMetadata(fullPath, nil, options.SetOwner)
+					remoteEntry.RestoreMetadata(fullPath, nil, metadataOptions)
 					if remoteEntry.IsHardLinkRoot() {
 						hardLinkTable[len(hardLinkTable)-1].willExist = true
 					}
@@ -876,7 +887,7 @@ func (manager *BackupManager) Restore(top string, revision int, options RestoreO
 				LOG_ERROR("RESTORE_SPECIAL", "Failed to restore special file %s: %v", fullPath, err)
 				return 0
 			}
-			remoteEntry.RestoreMetadata(fullPath, nil, options.SetOwner)
+			remoteEntry.RestoreMetadata(fullPath, nil, metadataOptions)
 			LOG_TRACE("DOWNLOAD_DONE", "Special %s %s restored", remoteEntry.Path, remoteEntry.FmtSpecial())
 
 		} else {
@@ -982,7 +993,7 @@ func (manager *BackupManager) Restore(top string, revision int, options RestoreO
 			}
 			newFile.Close()
 
-			file.RestoreMetadata(fullPath, nil, options.SetOwner)
+			file.RestoreMetadata(fullPath, nil, metadataOptions)
 			if !options.ShowStatistics {
 				LOG_INFO("DOWNLOAD_DONE", "Downloaded %s (0)", file.Path)
 				downloadedFileSize += file.Size
@@ -993,7 +1004,8 @@ func (manager *BackupManager) Restore(top string, revision int, options RestoreO
 		}
 
 		downloaded, err := manager.RestoreFile(chunkDownloader, chunkMaker, file, top, options.InPlace, overwrite,
-			options.ShowStatistics, totalFileSize, downloadedFileSize, startDownloadingTime, allowFailures)
+			options.ShowStatistics, totalFileSize, downloadedFileSize, startDownloadingTime, allowFailures,
+			metadataOptions.FileFlagsMask)
 		if err != nil {
 			// RestoreFile returned an error; if allowFailures is false RestoerFile would error out and not return so here
 			// we just need to show a warning
@@ -1012,7 +1024,7 @@ func (manager *BackupManager) Restore(top string, revision int, options RestoreO
 			skippedFileSize += file.Size
 			skippedFileCount++
 		}
-		file.RestoreMetadata(fullPath, nil, options.SetOwner)
+		file.RestoreMetadata(fullPath, nil, metadataOptions)
 	}
 
 	for _, linkEntry := range hardLinks {
@@ -1059,7 +1071,7 @@ func (manager *BackupManager) Restore(top string, revision int, options RestoreO
 
 	for _, entry := range directoryEntries {
 		dir := joinPath(top, entry.Path)
-		entry.RestoreMetadata(dir, nil, options.SetOwner)
+		entry.RestoreMetadata(dir, nil, metadataOptions)
 	}
 
 	if options.ShowStatistics {
@@ -1273,7 +1285,8 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
 //	false, nil:   Skipped file;
 //	false, error: Failure to restore file (only if allowFailures == true)
 func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chunkMaker *ChunkMaker, entry *Entry, top string, inPlace bool, overwrite bool,
-	showStatistics bool, totalFileSize int64, downloadedFileSize int64, startTime int64, allowFailures bool) (bool, error) {
+	showStatistics bool, totalFileSize int64, downloadedFileSize int64, startTime int64, allowFailures bool,
+	fileFlagsMask uint32) (bool, error) {
 
 	LOG_TRACE("DOWNLOAD_START", "Downloading %s", entry.Path)
 
@@ -1320,7 +1333,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 					LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing: %v", fullPath, err)
 					return false, nil
 				}
-				err = entry.RestoreEarlyFileFlags(existingFile, 0) // TODO: implement mask
+				err = entry.RestoreEarlyFileFlags(existingFile, fileFlagsMask)
 				if err != nil {
 					LOG_WARN("DOWNLOAD_FLAGS", "Failed to set early file flags on %s: %v", fullPath, err)
 				}
@@ -1507,7 +1520,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 				return false, nil
 			}
 		}
-		err = entry.RestoreEarlyFileFlags(existingFile, 0) // TODO: implement mask
+		err = entry.RestoreEarlyFileFlags(existingFile, fileFlagsMask)
 		if err != nil {
 			LOG_WARN("DOWNLOAD_FLAGS", "Failed to set early file flags on %s: %v", fullPath, err)
 		}
@@ -1593,7 +1606,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			LOG_ERROR("DOWNLOAD_OPEN", "Failed to open file for writing: %v", err)
 			return false, nil
 		}
-		err = entry.RestoreEarlyFileFlags(newFile, 0) // TODO: implement mask
+		err = entry.RestoreEarlyFileFlags(newFile, fileFlagsMask)
 		if err != nil {
 			LOG_WARN("DOWNLOAD_FLAGS", "Failed to set early file flags on %s: %v", fullPath, err)
 		}

src/duplicacy_entry.go

@@ -582,7 +582,15 @@ func (entry *Entry) String(maxSizeDigits int) string {
 	return fmt.Sprintf("%*d %s %64s %s", maxSizeDigits, entry.Size, modifiedTime, entry.Hash, entry.Path)
 }
 
-func (entry *Entry) RestoreMetadata(fullPath string, fileInfo os.FileInfo, setOwner bool) bool {
+type RestoreMetadataOptions struct {
+	SetOwner bool
+	ExcludeXattrs bool
+	NormalizeXattrs bool
+	FileFlagsMask uint32
+}
+
+func (entry *Entry) RestoreMetadata(fullPath string, fileInfo os.FileInfo,
+	options RestoreMetadataOptions) bool {
 
 	if fileInfo == nil {
 		var err error
@@ -593,13 +601,15 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo os.FileInfo, setOw
 		}
 	}
 
-	err := entry.SetAttributesToFile(fullPath)
-	if err != nil {
-		LOG_WARN("RESTORE_ATTR", "Failed to set extended attributes on %s: %v", entry.Path, err)
+	if !options.ExcludeXattrs {
+		err := entry.SetAttributesToFile(fullPath, options.NormalizeXattrs)
+		if err != nil {
+			LOG_WARN("RESTORE_ATTR", "Failed to set extended attributes on %s: %v", entry.Path, err)
+		}
 	}
 
 	// Note that chown can remove setuid/setgid bits so should be called before chmod
-	if setOwner {
+	if options.SetOwner {
 		if !SetOwner(fullPath, entry, fileInfo) {
 			return false
 		}
@@ -624,7 +634,7 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo os.FileInfo, setOw
 		}
 	}
 
-	err = entry.RestoreLateFileFlags(fullPath, fileInfo, 0) // TODO: implement mask
+	err := entry.RestoreLateFileFlags(fullPath, fileInfo, options.FileFlagsMask)
 	if err != nil {
 		LOG_WARN("RESTORE_FLAGS", "Failed to set file flags on %s: %v", entry.Path, err)
 	}

src/duplicacy_preference.go

@@ -6,27 +6,55 @@ package duplicacy
 
 import (
 	"encoding/json"
-	"io/ioutil"
+	"fmt"
 	"os"
 	"path"
 	"reflect"
+	"strconv"
 	"strings"
 )
 
+type flagsMask uint32
+
+func (f flagsMask) MarshalJSON() ([]byte, error) {
+	return json.Marshal(fmt.Sprintf("0x%.8x", f))
+}
+
+func (f *flagsMask) UnmarshalJSON(data []byte) error {
+	var str string
+	if err := json.Unmarshal(data, &str); err != nil {
+		return err
+	}
+	if str[0] == '0' && (str[1] == 'x' || str[1] == 'X') {
+		str = str[2:]
+	}
+
+	v, err := strconv.ParseUint(string(str), 16, 32)
+	if err != nil {
+		return err
+	}
+	*f = flagsMask(v)
+	return nil
+}
+
 // Preference stores options for each storage.
 type Preference struct {
-	Name              string            `json:"name"`
-	SnapshotID        string            `json:"id"`
-	RepositoryPath    string            `json:"repository"`
-	StorageURL        string            `json:"storage"`
-	Encrypted         bool              `json:"encrypted"`
-	BackupProhibited  bool              `json:"no_backup"`
-	RestoreProhibited bool              `json:"no_restore"`
-	DoNotSavePassword bool              `json:"no_save_password"`
-	NobackupFile      string            `json:"nobackup_file"`
-	Keys              map[string]string `json:"keys"`
-	FiltersFile       string            `json:"filters"`
-	ExcludeByAttribute bool             `json:"exclude_by_attribute"`
+	Name               string            `json:"name"`
+	SnapshotID         string            `json:"id"`
+	RepositoryPath     string            `json:"repository"`
+	StorageURL         string            `json:"storage"`
+	Encrypted          bool              `json:"encrypted"`
+	BackupProhibited   bool              `json:"no_backup"`
+	RestoreProhibited  bool              `json:"no_restore"`
+	DoNotSavePassword  bool              `json:"no_save_password"`
+	NobackupFile       string            `json:"nobackup_file"`
+	Keys               map[string]string `json:"keys"`
+	FiltersFile        string            `json:"filters"`
+	ExcludeByAttribute bool              `json:"exclude_by_attribute"`
+	ExcludeXattrs      bool              `json:"exclude_xattrs"`
+	NormalizeXattrs    bool              `json:"normalize_xattrs"`
+	IncludeSpecials    bool              `json:"include_specials"`
+	FileFlagsMask      flagsMask         `json:"file_flags_mask"`
 }
 
 var preferencePath string
@@ -43,7 +71,7 @@ func LoadPreferences(repository string) bool {
 	}
 
 	if !stat.IsDir() {
-		content, err := ioutil.ReadFile(preferencePath)
+		content, err := os.ReadFile(preferencePath)
 		if err != nil {
 			LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to locate the preference path: %v", err)
 			return false
@@ -61,7 +89,7 @@ func LoadPreferences(repository string) bool {
 		preferencePath = realPreferencePath
 	}
 
-	description, err := ioutil.ReadFile(path.Join(preferencePath, "preferences"))
+	description, err := os.ReadFile(path.Join(preferencePath, "preferences"))
 	if err != nil {
 		LOG_ERROR("PREFERENCE_OPEN", "Failed to read the preference file from repository %s: %v", repository, err)
 		return false
@@ -110,7 +138,7 @@ func SavePreferences() bool {
 	}
 	preferenceFile := path.Join(GetDuplicacyPreferencePath(), "preferences")
 
-	err = ioutil.WriteFile(preferenceFile, description, 0600)
+	err = os.WriteFile(preferenceFile, description, 0600)
 	if err != nil {
 		LOG_ERROR("PREFERENCE_WRITE", "Failed to save the preference file %s: %v", preferenceFile, err)
 		return false

src/duplicacy_utils_windows.go

@@ -114,6 +114,29 @@ func SetOwner(fullPath string, entry *Entry, fileInfo os.FileInfo) bool {
 	return true
 }
 
+func MakeHardlink(source string, target string) error {
+	return os.Link(source, target)
+}
+
+func joinPath(components ...string) string {
+
+	combinedPath := `\\?\` + filepath.Join(components...)
+	// If the path is on a samba drive we must use the UNC format
+	if strings.HasPrefix(combinedPath, `\\?\\\`) {
+		combinedPath = `\\?\UNC\` + combinedPath[6:]
+	}
+	return combinedPath
+}
+
+func SplitDir(fullPath string) (dir string, file string) {
+	i := strings.LastIndex(fullPath, "\\")
+	return fullPath[:i+1], fullPath[i+1:]
+}
+
+func excludedByAttribute(attributes map[string][]byte) bool {
+	return false
+}
+
 type listEntryLinkKey struct{}
 
 func (entry *Entry) getHardLinkKey(f os.FileInfo) (key listEntryLinkKey, linked bool) {
@@ -128,7 +151,7 @@ func (entry *Entry) ReadFileFlags(fullPath string, fileInfo os.FileInfo) error {
 	return nil
 }
 
-func (entry *Entry) SetAttributesToFile(fullPath string) error {
+func (entry *Entry) SetAttributesToFile(fullPath string, normalize bool) error {
 	return nil
 }
 
@@ -159,26 +182,3 @@ func (entry *Entry) RestoreSpecial(fullPath string) error {
 func (entry *Entry) FmtSpecial() string {
 	return ""
 }
-
-func MakeHardlink(source string, target string) error {
-	return os.Link(source, target)
-}
-
-func joinPath(components ...string) string {
-
-	combinedPath := `\\?\` + filepath.Join(components...)
-	// If the path is on a samba drive we must use the UNC format
-	if strings.HasPrefix(combinedPath, `\\?\\\`) {
-		combinedPath = `\\?\UNC\` + combinedPath[6:]
-	}
-	return combinedPath
-}
-
-func SplitDir(fullPath string) (dir string, file string) {
-	i := strings.LastIndex(fullPath, "\\")
-	return fullPath[:i+1], fullPath[i+1:]
-}
-
-func excludedByAttribute(attributes map[string][]byte) bool {
-	return false
-}

src/duplicacy_xattr_darwin.go

@@ -64,7 +64,7 @@ func (entry *Entry) ReadFileFlags(fullPath string, fileInfo os.FileInfo) error {
 	return nil
 }
 
-func (entry *Entry) SetAttributesToFile(fullPath string) error {
+func (entry *Entry) SetAttributesToFile(fullPath string, normalize bool) error {
 	if entry.Attributes == nil || len(*entry.Attributes) == 0 || entry.IsSpecial() {
 		return nil
 	}
@@ -109,7 +109,7 @@ func (entry *Entry) RestoreEarlyFileFlags(f *os.File, mask uint32) error {
 }
 
 func (entry *Entry) RestoreLateFileFlags(fullPath string, fileInfo os.FileInfo, mask uint32) error {
-	if entry.Attributes == nil {
+	if mask == 0xffffffff {
 		return nil
 	}
 
@@ -121,8 +121,10 @@ func (entry *Entry) RestoreLateFileFlags(fullPath string, fileInfo os.FileInfo,
 
 	var flags uint32
 
-	if v, have := (*entry.Attributes)[darwinFileFlagsKey]; have {
-		flags = binary.LittleEndian.Uint32(v)
+	if entry.Attributes != nil {
+		if v, have := (*entry.Attributes)[darwinFileFlagsKey]; have {
+			flags = binary.LittleEndian.Uint32(v)
+		}
 	}
 
 	stat := fileInfo.Sys().(*syscall.Stat_t)

src/duplicacy_xattr_linux.go

@@ -124,7 +124,7 @@ func (entry *Entry) ReadFileFlags(fullPath string, fileInfo os.FileInfo) error {
 	return nil
 }
 
-func (entry *Entry) SetAttributesToFile(fullPath string) error {
+func (entry *Entry) SetAttributesToFile(fullPath string, normalize bool) error {
 	if entry.Attributes == nil || len(*entry.Attributes) == 0 {
 		return nil
 	}
@@ -161,7 +161,7 @@ func (entry *Entry) SetAttributesToFile(fullPath string) error {
 }
 
 func (entry *Entry) RestoreEarlyDirFlags(fullPath string, mask uint32) error {
-	if entry.Attributes == nil {
+	if entry.Attributes == nil || mask == 0xffffffff {
 		return nil
 	}
 	var flags uint32
@@ -185,7 +185,7 @@ func (entry *Entry) RestoreEarlyDirFlags(fullPath string, mask uint32) error {
 }
 
 func (entry *Entry) RestoreEarlyFileFlags(f *os.File, mask uint32) error {
-	if entry.Attributes == nil {
+	if entry.Attributes == nil || mask == 0xffffffff {
 		return nil
 	}
 	var flags uint32
@@ -204,7 +204,7 @@ func (entry *Entry) RestoreEarlyFileFlags(f *os.File, mask uint32) error {
 }
 
 func (entry *Entry) RestoreLateFileFlags(fullPath string, fileInfo os.FileInfo, mask uint32) error {
-	if entry.IsLink() || entry.Attributes == nil {
+	if entry.IsLink() || entry.Attributes == nil || mask == 0xffffffff {
 		return nil
 	}
 	var flags uint32
@@ -218,7 +218,7 @@ func (entry *Entry) RestoreLateFileFlags(fullPath string, fileInfo os.FileInfo,
 		if err != nil {
 			return err
 		}
-		err = ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
+		err = ioctl(f, unix.FS_IOC_SETFLAGS, &flags)
 		f.Close()
 		if err != nil {
 			return fmt.Errorf("Set flags 0x%.8x failed: %w", flags, err)

src/duplicacy_xattr_xbsd.go

@@ -71,7 +71,7 @@ func (entry *Entry) ReadFileFlags(fullPath string, fileInfo os.FileInfo) error {
 	return nil
 }
 
-func (entry *Entry) SetAttributesToFile(fullPath string) error {
+func (entry *Entry) SetAttributesToFile(fullPath string, normalize bool) error {
 	if entry.Attributes == nil || len(*entry.Attributes) == 0 || entry.IsSpecial() {
 		return nil
 	}
@@ -116,7 +116,7 @@ func (entry *Entry) RestoreEarlyFileFlags(f *os.File, mask uint32) error {
 }
 
 func (entry *Entry) RestoreLateFileFlags(fullPath string, fileInfo os.FileInfo, mask uint32) error {
-	if entry.Attributes == nil {
+	if mask == 0xffffffff {
 		return nil
 	}
 
@@ -128,8 +128,10 @@ func (entry *Entry) RestoreLateFileFlags(fullPath string, fileInfo os.FileInfo,
 
 	var flags uint32
 
-	if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
-		flags = binary.LittleEndian.Uint32(v)
+	if entry.Attributes != nil {
+		if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
+			flags = binary.LittleEndian.Uint32(v)
+		}
 	}
 
 	stat := fileInfo.Sys().(*syscall.Stat_t)