Initial implementation of file/inode flags (Linux, BSD, darwin)

Basic support for BSD and Darwin style chflags (stat flags). Applies
these flags at the end of file restore.
Supports linux style ioctl_iflags(2) in a 2 step process. Flags that
need to be applied prior to writes such as compress and especially no-COW
are applied immediately upon file open.

The flags format is backwards compatible. An attribute starting with a
null byte is used to store flags in the entry attributes table. With
an old version of duplicacy the restore of this attribute should silently
fail (effectively be ignored).

Fixes xattr restore to use O_NOFOLLOW so attributes are applied to symlink.

TODO: Tests, possible option to switch off mutable/append prior to
restore of existing file similar to rsync. Does not apply attributes
or flags to the top most directory.

go.mod

@@ -15,7 +15,6 @@ require (
 	github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c
 	github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8
 	github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508
-	github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01
 	github.com/hirochachacha/go-smb2 v1.1.0
 	github.com/klauspost/compress v1.16.3
 	github.com/klauspost/reedsolomon v1.9.9

go.sum

@@ -47,8 +47,6 @@ github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8 h1:ijgl4Y+
 github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8/go.mod h1:0lQcVva56+L1PuUFXLOsJ6arJQaU0baIH8q+IegeBhg=
 github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508 h1:SqTyk5KkNXp7zTdTttIZSDcTrL5uau4K/2OpKvgBZVI=
 github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508/go.mod h1:w/pisxUZezf2XzU9Ewjphcf6q1mZtOzKPHhJiuc8cag=
-github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01 h1:LqwS9qL6SrDkp0g0iwUkETrDdtB9gTKaIbSn9imUq5o=
-github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01/go.mod h1:TMlibuxKfkdtHyltooAw7+DHqRpaXs9nxaffk00Sh1Q=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 h1:G1U0vew/vA/1/hBmf1XNeyIzJJbPFVv+kb+HPl6rj6c=

src/duplicacy_b2client.go

@@ -396,7 +396,7 @@ type B2ListFileNamesOutput struct {
 
 func (client *B2Client) ListFileNames(threadIndex int, startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
 
-	maxFileCount := 1000
+	maxFileCount := 10_000
 	if singleFile {
 		if includeVersions {
 			maxFileCount = 4

src/duplicacy_backupmanager.go

@@ -780,6 +780,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 					return 0
 				}
 			}
+			remoteEntry.RestoreEarlyDirFlags(fullPath)
 			directoryEntries = append(directoryEntries, remoteEntry)
 		} else {
 			// We can't download files here since fileEntries needs to be sorted
@@ -1194,6 +1195,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 					LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing: %v", fullPath, err)
 					return false, nil
 				}
+				entry.RestoreEarlyFileFlags(existingFile)
 
 				n := int64(1)
 				// There is a go bug on Windows (https://github.com/golang/go/issues/21681) that causes Seek to fail
@@ -1377,6 +1379,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 				return false, nil
 			}
 		}
+		entry.RestoreEarlyFileFlags(existingFile)
 
 		existingFile.Seek(0, 0)
 
@@ -1459,6 +1462,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			LOG_ERROR("DOWNLOAD_OPEN", "Failed to open file for writing: %v", err)
 			return false, nil
 		}
+		entry.RestoreEarlyFileFlags(newFile)
 
 		hasher := manager.config.NewFileHasher()
 

src/duplicacy_entry_test.go

@@ -5,6 +5,8 @@
 package duplicacy
 
 import (
+	"bytes"
+	"encoding/json"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -13,11 +15,10 @@ import (
 	"sort"
 	"strings"
 	"testing"
-	"bytes"
-	"encoding/json"
 
-	"github.com/gilbertchen/xattr"
-    "github.com/vmihailenco/msgpack"
+	"github.com/pkg/xattr"
+
+	"github.com/vmihailenco/msgpack"
 )
 
 func TestEntrySort(t *testing.T) {
@@ -175,7 +176,7 @@ func TestEntryOrder(t *testing.T) {
 	directories = append(directories, CreateEntry("", 0, 0, 0))
 
 	entries := make([]*Entry, 0, 4)
-    entryChannel := make(chan *Entry, 1024)
+	entryChannel := make(chan *Entry, 1024)
 	entries = append(entries, CreateEntry("", 0, 0, 0))
 
 	for len(directories) > 0 {
@@ -233,8 +234,16 @@ func TestEntryOrder(t *testing.T) {
 // TestEntryExcludeByAttribute tests the excludeByAttribute parameter to the ListEntries function
 func TestEntryExcludeByAttribute(t *testing.T) {
 
-	if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") {
-		t.Skip("skipping test not darwin or linux")
+	var excludeAttrName string
+	var excludeAttrValue []byte
+
+	if runtime.GOOS == "darwin" {
+		excludeAttrName = "com.apple.metadata:com_apple_backup_excludeItem"
+		excludeAttrValue = []byte("com.apple.backupd")
+	} else if runtime.GOOS == "linux" || runtime.GOOS == "freebsd" || runtime.GOOS == "netbsd" || runtime.GOOS == "solaris" {
+		excludeAttrName = "user.duplicacy_exclude"
+	} else {
+		t.Skip("skipping test, not darwin, linux, freebsd, netbsd, or solaris")
 	}
 
 	testDir := filepath.Join(os.TempDir(), "duplicacy_test")
@@ -273,7 +282,7 @@ func TestEntryExcludeByAttribute(t *testing.T) {
 	for _, file := range DATA {
 		fullPath := filepath.Join(testDir, file)
 		if strings.Contains(file, "exclude") {
-			xattr.Setxattr(fullPath, "com.apple.metadata:com_apple_backup_excludeItem", []byte("com.apple.backupd"))
+			xattr.Set(fullPath, excludeAttrName, excludeAttrValue)
 		}
 	}
 
@@ -372,4 +381,4 @@ func TestEntryEncoding(t *testing.T) {
 		t.Error("Decoded entry is different than the original one")
 	}
 
-}
+}

src/duplicacy_s3storage.go

@@ -90,48 +90,40 @@ func (storage *S3Storage) ListFiles(threadIndex int, dir string) (files []string
 
 	if dir == "snapshots/" {
 		dir = storage.storageDir + dir
-		input := s3.ListObjectsInput{
+		input := s3.ListObjectsV2Input{
 			Bucket:    aws.String(storage.bucket),
 			Prefix:    aws.String(dir),
 			Delimiter: aws.String("/"),
-			MaxKeys:   aws.Int64(1000),
 		}
 
-		output, err := storage.client.ListObjects(&input)
+		err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+			for _, subDir := range page.CommonPrefixes {
+				files = append(files, (*subDir.Prefix)[len(dir):])
+			}
+			return true
+		})
 		if err != nil {
 			return nil, nil, err
 		}
 
-		for _, subDir := range output.CommonPrefixes {
-			files = append(files, (*subDir.Prefix)[len(dir):])
-		}
 		return files, nil, nil
 	} else {
 		dir = storage.storageDir + dir
-		marker := ""
-		for {
-			input := s3.ListObjectsInput{
-				Bucket:  aws.String(storage.bucket),
-				Prefix:  aws.String(dir),
-				MaxKeys: aws.Int64(1000),
-				Marker:  aws.String(marker),
-			}
+		input := s3.ListObjectsV2Input{
+			Bucket:  aws.String(storage.bucket),
+			Prefix:  aws.String(dir),
+			MaxKeys: aws.Int64(1000),
+		}
 
-			output, err := storage.client.ListObjects(&input)
-			if err != nil {
-				return nil, nil, err
-			}
-
-			for _, object := range output.Contents {
+		err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+			for _, object := range page.Contents {
 				files = append(files, (*object.Key)[len(dir):])
 				sizes = append(sizes, *object.Size)
 			}
-
-			if !*output.IsTruncated {
-				break
-			}
-
-			marker = *output.Contents[len(output.Contents)-1].Key
+			return true
+		})
+		if err != nil {
+			return nil, nil, err
 		}
 		return files, sizes, nil
 	}

src/duplicacy_utils_bsd.go

@@ -0,0 +1,53 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+//go:build freebsd || netbsd || darwin
+// +build freebsd netbsd darwin
+
+package duplicacy
+
+import (
+	"encoding/binary"
+	"os"
+	"syscall"
+)
+
+const bsdFileFlagsKey = "\x00bf"
+
+func (entry *Entry) ReadFileFlags(f *os.File) error {
+	fileInfo, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	stat, ok := fileInfo.Sys().(*syscall.Stat_t)
+	if ok && stat.Flags != 0 {
+		if entry.Attributes == nil {
+			entry.Attributes = &map[string][]byte{}
+		}
+		v := make([]byte, 4)
+		binary.LittleEndian.PutUint32(v, stat.Flags)
+		(*entry.Attributes)[bsdFileFlagsKey] = v
+		LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", stat.Flags, entry.Path)
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyDirFlags(path string) error {
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
+	return nil
+}
+
+func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
+		LOG_DEBUG("ATTR_RESTORE", "Restore flags 0x%x for %s", binary.LittleEndian.Uint32(v), entry.Path)
+		return syscall.Fchflags(int(f.Fd()), int(binary.LittleEndian.Uint32(v)))
+	}
+	return nil
+}

src/duplicacy_utils_darwin.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 )
 
-func excludedByAttribute(attirbutes map[string][]byte) bool {
-	value, ok := attirbutes["com.apple.metadata:com_apple_backup_excludeItem"]
+func excludedByAttribute(attributes map[string][]byte) bool {
+	value, ok := attributes["com.apple.metadata:com_apple_backup_excludeItem"]
 	return ok && strings.Contains(string(value), "com.apple.backupd")
 }

src/duplicacy_utils_linux.go

@@ -5,9 +5,108 @@
 package duplicacy
 
 import (
+	"encoding/binary"
+	"os"
+	"syscall"
+	"unsafe"
 )
 
-func excludedByAttribute(attirbutes map[string][]byte) bool {
-	_, ok := attirbutes["duplicacy_exclude"]
+const (
+	linux_FS_SECRM_FL        = 0x00000001 /* Secure deletion */
+	linux_FS_UNRM_FL         = 0x00000002 /* Undelete */
+	linux_FS_COMPR_FL        = 0x00000004 /* Compress file */
+	linux_FS_SYNC_FL         = 0x00000008 /* Synchronous updates */
+	linux_FS_IMMUTABLE_FL    = 0x00000010 /* Immutable file */
+	linux_FS_APPEND_FL       = 0x00000020 /* writes to file may only append */
+	linux_FS_NODUMP_FL       = 0x00000040 /* do not dump file */
+	linux_FS_NOATIME_FL      = 0x00000080 /* do not update atime */
+	linux_FS_NOCOMP_FL       = 0x00000400 /* Don't compress */
+	linux_FS_JOURNAL_DATA_FL = 0x00004000 /* Reserved for ext3 */
+	linux_FS_NOTAIL_FL       = 0x00008000 /* file tail should not be merged */
+	linux_FS_DIRSYNC_FL      = 0x00010000 /* dirsync behaviour (directories only) */
+	linux_FS_TOPDIR_FL       = 0x00020000 /* Top of directory hierarchies*/
+	linux_FS_NOCOW_FL        = 0x00800000 /* Do not cow file */
+	linux_FS_PROJINHERIT_FL  = 0x20000000 /* Create with parents projid */
+
+	linux_FS_IOC_GETFLAGS uintptr = 0x80086601
+	linux_FS_IOC_SETFLAGS uintptr = 0x40086602
+
+	linuxIocFlagsFileEarly = linux_FS_SECRM_FL | linux_FS_UNRM_FL | linux_FS_COMPR_FL | linux_FS_NODUMP_FL | linux_FS_NOATIME_FL | linux_FS_NOCOMP_FL | linux_FS_JOURNAL_DATA_FL | linux_FS_NOTAIL_FL | linux_FS_NOCOW_FL
+	linuxIocFlagsDirEarly  = linux_FS_TOPDIR_FL | linux_FS_PROJINHERIT_FL
+	linuxIocFlagsLate      = linux_FS_SYNC_FL | linux_FS_IMMUTABLE_FL | linux_FS_APPEND_FL | linux_FS_DIRSYNC_FL
+
+	linuxFileFlagsKey = "\x00lf"
+)
+
+func ioctl(f *os.File, request uintptr, attrp *uint32) error {
+	argp := uintptr(unsafe.Pointer(attrp))
+
+	if _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), request, argp); errno != 0 {
+		return os.NewSyscallError("ioctl", errno)
+	}
+	return nil
+}
+
+func (entry *Entry) ReadFileFlags(f *os.File) error {
+	var flags uint32
+	if err := ioctl(f, linux_FS_IOC_GETFLAGS, &flags); err != nil {
+		return err
+	}
+	if flags != 0 {
+		if entry.Attributes == nil {
+			entry.Attributes = &map[string][]byte{}
+		}
+		v := make([]byte, 4)
+		binary.LittleEndian.PutUint32(v, flags)
+		(*entry.Attributes)[linuxFileFlagsKey] = v
+		LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", flags, entry.Path)
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyDirFlags(path string) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
+		flags := binary.LittleEndian.Uint32(v) & linuxIocFlagsDirEarly
+		f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_DIRECTORY, 0)
+		if err != nil {
+			return err
+		}
+		LOG_DEBUG("ATTR_RESTORE", "Restore dir flags (early) 0x%x for %s", flags, entry.Path)
+		err = ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
+		f.Close()
+		return err
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
+		flags := binary.LittleEndian.Uint32(v) & linuxIocFlagsFileEarly
+		LOG_DEBUG("ATTR_RESTORE", "Restore flags (early) 0x%x for %s", flags, entry.Path)
+		return ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
+		flags := binary.LittleEndian.Uint32(v) & (linuxIocFlagsFileEarly | linuxIocFlagsDirEarly | linuxIocFlagsLate)
+		LOG_DEBUG("ATTR_RESTORE", "Restore flags (late) 0x%x for %s", flags, entry.Path)
+		return ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
+	}
+	return nil
+}
+
+func excludedByAttribute(attributes map[string][]byte) bool {
+	_, ok := attributes["user.duplicacy_exclude"]
 	return ok
 }

src/duplicacy_utils_others.go

@@ -48,9 +48,12 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
 }
 
 func (entry *Entry) ReadAttributes(top string) {
-
 	fullPath := filepath.Join(top, entry.Path)
-	attributes, _ := xattr.List(fullPath)
+	f, err := os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
+	if err != nil {
+		return
+	}
+	attributes, _ := xattr.FList(f)
 	if len(attributes) > 0 {
 		entry.Attributes = &map[string][]byte{}
 		for _, name := range attributes {
@@ -60,30 +63,42 @@ func (entry *Entry) ReadAttributes(top string) {
 			}
 		}
 	}
+	if err := entry.ReadFileFlags(f); err != nil {
+		LOG_INFO("ATTR_BACKUP", "Could not backup flags for file %s: %v", fullPath, err)
+	}
+	f.Close()
 }
 
 func (entry *Entry) SetAttributesToFile(fullPath string) {
-	names, _ := xattr.List(fullPath)
+	f, err := os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
+	if err != nil {
+		return
+	}
 
+	names, _ := xattr.FList(f)
 	for _, name := range names {
-
-
 		newAttribute, found := (*entry.Attributes)[name]
 		if found {
-			oldAttribute, _ := xattr.Get(fullPath, name)
+			oldAttribute, _ := xattr.FGet(f, name)
 			if !bytes.Equal(oldAttribute, newAttribute) {
-				xattr.Set(fullPath, name, newAttribute)
+				xattr.FSet(f, name, newAttribute)
 			}
 			delete(*entry.Attributes, name)
 		} else {
-			xattr.Remove(fullPath, name)
+			xattr.FRemove(f, name)
 		}
 	}
 
 	for name, attribute := range *entry.Attributes {
-		xattr.Set(fullPath, name, attribute)
+		if len(name) > 0 && name[0] == '\x00' {
+			continue
+		}
+		xattr.FSet(f, name, attribute)
 	}
-
+	if err := entry.RestoreLateFileFlags(f); err != nil {
+		LOG_DEBUG("ATTR_RESTORE", "Could not restore flags for file %s: %v", fullPath, err)
+	}
+	f.Close()
 }
 
 func joinPath(components ...string) string {

src/duplicacy_utils_posix.go

@@ -2,12 +2,12 @@
 // Free for personal use and commercial trial
 // Commercial use requires per-user licenses available from https://duplicacy.com
 
+//go:build freebsd || netbsd || solaris
+// +build freebsd netbsd solaris
+
 package duplicacy
 
-import (
-)
-
-func excludedByAttribute(attirbutes map[string][]byte) bool {
-	_, ok := attirbutes["duplicacy_exclude"]
+func excludedByAttribute(attributes map[string][]byte) bool {
+	_, ok := attributes["user.duplicacy_exclude"]
 	return ok
-}
+}

src/duplicacy_utils_windows.go

@@ -132,6 +132,18 @@ func SplitDir(fullPath string) (dir string, file string) {
 	return fullPath[:i+1], fullPath[i+1:]
 }
 
-func excludedByAttribute(attirbutes map[string][]byte) bool {
-	return false
+func (entry *Entry) ReadFileFlags(f *os.File) error {
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyDirFlags(path string) error {
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
+	return nil
+}
+
+func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
+	return nil
 }