Use S3 ListObjectsV2 for listing files

ListObjects has been deprecated since 2016 and ListObjectsV2 with use of
explicit pagination tokens is more performant for large listings as well.

This also mitigates an issue with iDrive E2 where the StartAfter/Marker
is included in the output, leading to duplicate entries. Right now this
causes an exhaustive prune to delete chunks erroneously flagged as
duplicate, destroying the storage.

duplicacy/duplicacy_main.go

@@ -2262,7 +2262,7 @@ func main() {
 	app.Name = "duplicacy"
 	app.HelpName = "duplicacy"
 	app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
-	app.Version = "3.2.1" + " (" + GitCommit + ")"
+	app.Version = "3.2.0" + " (" + GitCommit + ")"
 
 	// Exit with code 2 if an invalid command is provided
 	app.CommandNotFound = func(context *cli.Context, command string) {

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c
 	github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8
 	github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508
+	github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01
 	github.com/hirochachacha/go-smb2 v1.1.0
 	github.com/klauspost/compress v1.16.3
 	github.com/klauspost/reedsolomon v1.9.9

go.sum

@@ -47,6 +47,8 @@ github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8 h1:ijgl4Y+
 github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8/go.mod h1:0lQcVva56+L1PuUFXLOsJ6arJQaU0baIH8q+IegeBhg=
 github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508 h1:SqTyk5KkNXp7zTdTttIZSDcTrL5uau4K/2OpKvgBZVI=
 github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508/go.mod h1:w/pisxUZezf2XzU9Ewjphcf6q1mZtOzKPHhJiuc8cag=
+github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01 h1:LqwS9qL6SrDkp0g0iwUkETrDdtB9gTKaIbSn9imUq5o=
+github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01/go.mod h1:TMlibuxKfkdtHyltooAw7+DHqRpaXs9nxaffk00Sh1Q=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 h1:G1U0vew/vA/1/hBmf1XNeyIzJJbPFVv+kb+HPl6rj6c=

src/duplicacy_b2client.go

@@ -396,7 +396,7 @@ type B2ListFileNamesOutput struct {
 
 func (client *B2Client) ListFileNames(threadIndex int, startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
 
-	maxFileCount := 10_000
+	maxFileCount := 1000
 	if singleFile {
 		if includeVersions {
 			maxFileCount = 4

src/duplicacy_backupmanager.go

@@ -780,7 +780,6 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 					return 0
 				}
 			}
-			remoteEntry.RestoreEarlyDirFlags(fullPath)
 			directoryEntries = append(directoryEntries, remoteEntry)
 		} else {
 			// We can't download files here since fileEntries needs to be sorted
@@ -1195,7 +1194,6 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 					LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing: %v", fullPath, err)
 					return false, nil
 				}
-				entry.RestoreEarlyFileFlags(existingFile)
 
 				n := int64(1)
 				// There is a go bug on Windows (https://github.com/golang/go/issues/21681) that causes Seek to fail
@@ -1379,7 +1377,6 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 				return false, nil
 			}
 		}
-		entry.RestoreEarlyFileFlags(existingFile)
 
 		existingFile.Seek(0, 0)
 
@@ -1462,7 +1459,6 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			LOG_ERROR("DOWNLOAD_OPEN", "Failed to open file for writing: %v", err)
 			return false, nil
 		}
-		entry.RestoreEarlyFileFlags(newFile)
 
 		hasher := manager.config.NewFileHasher()
 

src/duplicacy_entry_test.go

@@ -5,8 +5,6 @@
 package duplicacy
 
 import (
-	"bytes"
-	"encoding/json"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -15,10 +13,11 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"bytes"
+	"encoding/json"
 
-	"github.com/pkg/xattr"
-
-	"github.com/vmihailenco/msgpack"
+	"github.com/gilbertchen/xattr"
+    "github.com/vmihailenco/msgpack"
 )
 
 func TestEntrySort(t *testing.T) {
@@ -176,7 +175,7 @@ func TestEntryOrder(t *testing.T) {
 	directories = append(directories, CreateEntry("", 0, 0, 0))
 
 	entries := make([]*Entry, 0, 4)
-	entryChannel := make(chan *Entry, 1024)
+    entryChannel := make(chan *Entry, 1024)
 	entries = append(entries, CreateEntry("", 0, 0, 0))
 
 	for len(directories) > 0 {
@@ -234,16 +233,8 @@ func TestEntryOrder(t *testing.T) {
 // TestEntryExcludeByAttribute tests the excludeByAttribute parameter to the ListEntries function
 func TestEntryExcludeByAttribute(t *testing.T) {
 
-	var excludeAttrName string
-	var excludeAttrValue []byte
-
-	if runtime.GOOS == "darwin" {
-		excludeAttrName = "com.apple.metadata:com_apple_backup_excludeItem"
-		excludeAttrValue = []byte("com.apple.backupd")
-	} else if runtime.GOOS == "linux" || runtime.GOOS == "freebsd" || runtime.GOOS == "netbsd" || runtime.GOOS == "solaris" {
-		excludeAttrName = "user.duplicacy_exclude"
-	} else {
-		t.Skip("skipping test, not darwin, linux, freebsd, netbsd, or solaris")
+	if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") {
+		t.Skip("skipping test not darwin or linux")
 	}
 
 	testDir := filepath.Join(os.TempDir(), "duplicacy_test")
@@ -282,7 +273,7 @@ func TestEntryExcludeByAttribute(t *testing.T) {
 	for _, file := range DATA {
 		fullPath := filepath.Join(testDir, file)
 		if strings.Contains(file, "exclude") {
-			xattr.Set(fullPath, excludeAttrName, excludeAttrValue)
+			xattr.Setxattr(fullPath, "com.apple.metadata:com_apple_backup_excludeItem", []byte("com.apple.backupd"))
 		}
 	}
 

src/duplicacy_utils_bsd.go

@@ -1,53 +0,0 @@
-// Copyright (c) Acrosync LLC. All rights reserved.
-// Free for personal use and commercial trial
-// Commercial use requires per-user licenses available from https://duplicacy.com
-
-//go:build freebsd || netbsd || darwin
-// +build freebsd netbsd darwin
-
-package duplicacy
-
-import (
-	"encoding/binary"
-	"os"
-	"syscall"
-)
-
-const bsdFileFlagsKey = "\x00bf"
-
-func (entry *Entry) ReadFileFlags(f *os.File) error {
-	fileInfo, err := f.Stat()
-	if err != nil {
-		return err
-	}
-	stat, ok := fileInfo.Sys().(*syscall.Stat_t)
-	if ok && stat.Flags != 0 {
-		if entry.Attributes == nil {
-			entry.Attributes = &map[string][]byte{}
-		}
-		v := make([]byte, 4)
-		binary.LittleEndian.PutUint32(v, stat.Flags)
-		(*entry.Attributes)[bsdFileFlagsKey] = v
-		LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", stat.Flags, entry.Path)
-	}
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyDirFlags(path string) error {
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
-	return nil
-}
-
-func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
-	if entry.Attributes == nil {
-		return nil
-	}
-	if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
-		LOG_DEBUG("ATTR_RESTORE", "Restore flags 0x%x for %s", binary.LittleEndian.Uint32(v), entry.Path)
-		return syscall.Fchflags(int(f.Fd()), int(binary.LittleEndian.Uint32(v)))
-	}
-	return nil
-}

src/duplicacy_utils_darwin.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 )
 
-func excludedByAttribute(attributes map[string][]byte) bool {
-	value, ok := attributes["com.apple.metadata:com_apple_backup_excludeItem"]
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	value, ok := attirbutes["com.apple.metadata:com_apple_backup_excludeItem"]
 	return ok && strings.Contains(string(value), "com.apple.backupd")
 }

src/duplicacy_utils_freebsd.go

@@ -2,12 +2,12 @@
 // Free for personal use and commercial trial
 // Commercial use requires per-user licenses available from https://duplicacy.com
 
-//go:build freebsd || netbsd || solaris
-// +build freebsd netbsd solaris
-
 package duplicacy
 
-func excludedByAttribute(attributes map[string][]byte) bool {
-	_, ok := attributes["user.duplicacy_exclude"]
+import (
+)
+
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	_, ok := attirbutes["duplicacy_exclude"]
 	return ok
 }

src/duplicacy_utils_linux.go

@@ -5,108 +5,9 @@
 package duplicacy
 
 import (
-	"encoding/binary"
-	"os"
-	"syscall"
-	"unsafe"
 )
 
-const (
-	linux_FS_SECRM_FL        = 0x00000001 /* Secure deletion */
-	linux_FS_UNRM_FL         = 0x00000002 /* Undelete */
-	linux_FS_COMPR_FL        = 0x00000004 /* Compress file */
-	linux_FS_SYNC_FL         = 0x00000008 /* Synchronous updates */
-	linux_FS_IMMUTABLE_FL    = 0x00000010 /* Immutable file */
-	linux_FS_APPEND_FL       = 0x00000020 /* writes to file may only append */
-	linux_FS_NODUMP_FL       = 0x00000040 /* do not dump file */
-	linux_FS_NOATIME_FL      = 0x00000080 /* do not update atime */
-	linux_FS_NOCOMP_FL       = 0x00000400 /* Don't compress */
-	linux_FS_JOURNAL_DATA_FL = 0x00004000 /* Reserved for ext3 */
-	linux_FS_NOTAIL_FL       = 0x00008000 /* file tail should not be merged */
-	linux_FS_DIRSYNC_FL      = 0x00010000 /* dirsync behaviour (directories only) */
-	linux_FS_TOPDIR_FL       = 0x00020000 /* Top of directory hierarchies*/
-	linux_FS_NOCOW_FL        = 0x00800000 /* Do not cow file */
-	linux_FS_PROJINHERIT_FL  = 0x20000000 /* Create with parents projid */
-
-	linux_FS_IOC_GETFLAGS uintptr = 0x80086601
-	linux_FS_IOC_SETFLAGS uintptr = 0x40086602
-
-	linuxIocFlagsFileEarly = linux_FS_SECRM_FL | linux_FS_UNRM_FL | linux_FS_COMPR_FL | linux_FS_NODUMP_FL | linux_FS_NOATIME_FL | linux_FS_NOCOMP_FL | linux_FS_JOURNAL_DATA_FL | linux_FS_NOTAIL_FL | linux_FS_NOCOW_FL
-	linuxIocFlagsDirEarly  = linux_FS_TOPDIR_FL | linux_FS_PROJINHERIT_FL
-	linuxIocFlagsLate      = linux_FS_SYNC_FL | linux_FS_IMMUTABLE_FL | linux_FS_APPEND_FL | linux_FS_DIRSYNC_FL
-
-	linuxFileFlagsKey = "\x00lf"
-)
-
-func ioctl(f *os.File, request uintptr, attrp *uint32) error {
-	argp := uintptr(unsafe.Pointer(attrp))
-
-	if _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), request, argp); errno != 0 {
-		return os.NewSyscallError("ioctl", errno)
-	}
-	return nil
-}
-
-func (entry *Entry) ReadFileFlags(f *os.File) error {
-	var flags uint32
-	if err := ioctl(f, linux_FS_IOC_GETFLAGS, &flags); err != nil {
-		return err
-	}
-	if flags != 0 {
-		if entry.Attributes == nil {
-			entry.Attributes = &map[string][]byte{}
-		}
-		v := make([]byte, 4)
-		binary.LittleEndian.PutUint32(v, flags)
-		(*entry.Attributes)[linuxFileFlagsKey] = v
-		LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", flags, entry.Path)
-	}
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyDirFlags(path string) error {
-	if entry.Attributes == nil {
-		return nil
-	}
-	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
-		flags := binary.LittleEndian.Uint32(v) & linuxIocFlagsDirEarly
-		f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_DIRECTORY, 0)
-		if err != nil {
-			return err
-		}
-		LOG_DEBUG("ATTR_RESTORE", "Restore dir flags (early) 0x%x for %s", flags, entry.Path)
-		err = ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
-		f.Close()
-		return err
-	}
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
-	if entry.Attributes == nil {
-		return nil
-	}
-	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
-		flags := binary.LittleEndian.Uint32(v) & linuxIocFlagsFileEarly
-		LOG_DEBUG("ATTR_RESTORE", "Restore flags (early) 0x%x for %s", flags, entry.Path)
-		return ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
-	}
-	return nil
-}
-
-func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
-	if entry.Attributes == nil {
-		return nil
-	}
-	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
-		flags := binary.LittleEndian.Uint32(v) & (linuxIocFlagsFileEarly | linuxIocFlagsDirEarly | linuxIocFlagsLate)
-		LOG_DEBUG("ATTR_RESTORE", "Restore flags (late) 0x%x for %s", flags, entry.Path)
-		return ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
-	}
-	return nil
-}
-
-func excludedByAttribute(attributes map[string][]byte) bool {
-	_, ok := attributes["user.duplicacy_exclude"]
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	_, ok := attirbutes["duplicacy_exclude"]
 	return ok
 }

src/duplicacy_utils_others.go

@@ -48,12 +48,9 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
 }
 
 func (entry *Entry) ReadAttributes(top string) {
+
 	fullPath := filepath.Join(top, entry.Path)
-	f, err := os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
-	if err != nil {
-		return
-	}
-	attributes, _ := xattr.FList(f)
+	attributes, _ := xattr.List(fullPath)
 	if len(attributes) > 0 {
 		entry.Attributes = &map[string][]byte{}
 		for _, name := range attributes {
@@ -63,42 +60,30 @@ func (entry *Entry) ReadAttributes(top string) {
 			}
 		}
 	}
-	if err := entry.ReadFileFlags(f); err != nil {
-		LOG_INFO("ATTR_BACKUP", "Could not backup flags for file %s: %v", fullPath, err)
-	}
-	f.Close()
 }
 
 func (entry *Entry) SetAttributesToFile(fullPath string) {
-	f, err := os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
-	if err != nil {
-		return
-	}
+	names, _ := xattr.List(fullPath)
 
-	names, _ := xattr.FList(f)
 	for _, name := range names {
+
+
 		newAttribute, found := (*entry.Attributes)[name]
 		if found {
-			oldAttribute, _ := xattr.FGet(f, name)
+			oldAttribute, _ := xattr.Get(fullPath, name)
 			if !bytes.Equal(oldAttribute, newAttribute) {
-				xattr.FSet(f, name, newAttribute)
+				xattr.Set(fullPath, name, newAttribute)
 			}
 			delete(*entry.Attributes, name)
 		} else {
-			xattr.FRemove(f, name)
+			xattr.Remove(fullPath, name)
 		}
 	}
 
 	for name, attribute := range *entry.Attributes {
-		if len(name) > 0 && name[0] == '\x00' {
-			continue
-		}
-		xattr.FSet(f, name, attribute)
+		xattr.Set(fullPath, name, attribute)
 	}
-	if err := entry.RestoreLateFileFlags(f); err != nil {
-		LOG_DEBUG("ATTR_RESTORE", "Could not restore flags for file %s: %v", fullPath, err)
-	}
-	f.Close()
+
 }
 
 func joinPath(components ...string) string {

src/duplicacy_utils_windows.go

@@ -132,18 +132,6 @@ func SplitDir(fullPath string) (dir string, file string) {
 	return fullPath[:i+1], fullPath[i+1:]
 }
 
-func (entry *Entry) ReadFileFlags(f *os.File) error {
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyDirFlags(path string) error {
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
-	return nil
-}
-
-func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
-	return nil
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	return false
 }