Bump version to 3.2.2

duplicacy/duplicacy_main.go

@@ -2262,7 +2262,7 @@ func main() {
 	app.Name = "duplicacy"
 	app.HelpName = "duplicacy"
 	app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
-	app.Version = "3.2.1" + " (" + GitCommit + ")"
+	app.Version = "3.2.2" + " (" + GitCommit + ")"
 
 	// Exit with code 2 if an invalid command is provided
 	app.CommandNotFound = func(context *cli.Context, command string) {

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c
 	github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8
 	github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508
+	github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01
 	github.com/hirochachacha/go-smb2 v1.1.0
 	github.com/klauspost/compress v1.16.3
 	github.com/klauspost/reedsolomon v1.9.9

go.sum

@@ -47,6 +47,8 @@ github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8 h1:ijgl4Y+
 github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8/go.mod h1:0lQcVva56+L1PuUFXLOsJ6arJQaU0baIH8q+IegeBhg=
 github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508 h1:SqTyk5KkNXp7zTdTttIZSDcTrL5uau4K/2OpKvgBZVI=
 github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508/go.mod h1:w/pisxUZezf2XzU9Ewjphcf6q1mZtOzKPHhJiuc8cag=
+github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01 h1:LqwS9qL6SrDkp0g0iwUkETrDdtB9gTKaIbSn9imUq5o=
+github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01/go.mod h1:TMlibuxKfkdtHyltooAw7+DHqRpaXs9nxaffk00Sh1Q=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 h1:G1U0vew/vA/1/hBmf1XNeyIzJJbPFVv+kb+HPl6rj6c=

src/duplicacy_b2client.go

@@ -396,7 +396,7 @@ type B2ListFileNamesOutput struct {
 
 func (client *B2Client) ListFileNames(threadIndex int, startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
 
-	maxFileCount := 10_000
+	maxFileCount := 1000
 	if singleFile {
 		if includeVersions {
 			maxFileCount = 4

src/duplicacy_backupmanager.go

@@ -780,7 +780,6 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 					return 0
 				}
 			}
-			remoteEntry.RestoreEarlyDirFlags(fullPath)
 			directoryEntries = append(directoryEntries, remoteEntry)
 		} else {
 			// We can't download files here since fileEntries needs to be sorted
@@ -1195,7 +1194,6 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 					LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing: %v", fullPath, err)
 					return false, nil
 				}
-				entry.RestoreEarlyFileFlags(existingFile)
 
 				n := int64(1)
 				// There is a go bug on Windows (https://github.com/golang/go/issues/21681) that causes Seek to fail
@@ -1379,7 +1377,6 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 				return false, nil
 			}
 		}
-		entry.RestoreEarlyFileFlags(existingFile)
 
 		existingFile.Seek(0, 0)
 
@@ -1462,7 +1459,6 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			LOG_ERROR("DOWNLOAD_OPEN", "Failed to open file for writing: %v", err)
 			return false, nil
 		}
-		entry.RestoreEarlyFileFlags(newFile)
 
 		hasher := manager.config.NewFileHasher()
 

src/duplicacy_entry_test.go

@@ -5,8 +5,6 @@
 package duplicacy
 
 import (
-	"bytes"
-	"encoding/json"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -15,10 +13,11 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"bytes"
+	"encoding/json"
 
-	"github.com/pkg/xattr"
-
-	"github.com/vmihailenco/msgpack"
+	"github.com/gilbertchen/xattr"
+    "github.com/vmihailenco/msgpack"
 )
 
 func TestEntrySort(t *testing.T) {
@@ -176,7 +175,7 @@ func TestEntryOrder(t *testing.T) {
 	directories = append(directories, CreateEntry("", 0, 0, 0))
 
 	entries := make([]*Entry, 0, 4)
-	entryChannel := make(chan *Entry, 1024)
+    entryChannel := make(chan *Entry, 1024)
 	entries = append(entries, CreateEntry("", 0, 0, 0))
 
 	for len(directories) > 0 {
@@ -234,16 +233,8 @@ func TestEntryOrder(t *testing.T) {
 // TestEntryExcludeByAttribute tests the excludeByAttribute parameter to the ListEntries function
 func TestEntryExcludeByAttribute(t *testing.T) {
 
-	var excludeAttrName string
-	var excludeAttrValue []byte
-
-	if runtime.GOOS == "darwin" {
-		excludeAttrName = "com.apple.metadata:com_apple_backup_excludeItem"
-		excludeAttrValue = []byte("com.apple.backupd")
-	} else if runtime.GOOS == "linux" || runtime.GOOS == "freebsd" || runtime.GOOS == "netbsd" || runtime.GOOS == "solaris" {
-		excludeAttrName = "user.duplicacy_exclude"
-	} else {
-		t.Skip("skipping test, not darwin, linux, freebsd, netbsd, or solaris")
+	if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") {
+		t.Skip("skipping test not darwin or linux")
 	}
 
 	testDir := filepath.Join(os.TempDir(), "duplicacy_test")
@@ -282,7 +273,7 @@ func TestEntryExcludeByAttribute(t *testing.T) {
 	for _, file := range DATA {
 		fullPath := filepath.Join(testDir, file)
 		if strings.Contains(file, "exclude") {
-			xattr.Set(fullPath, excludeAttrName, excludeAttrValue)
+			xattr.Setxattr(fullPath, "com.apple.metadata:com_apple_backup_excludeItem", []byte("com.apple.backupd"))
 		}
 	}
 
@@ -381,4 +372,4 @@ func TestEntryEncoding(t *testing.T) {
 		t.Error("Decoded entry is different than the original one")
 	}
 
-}
+}

src/duplicacy_s3storage.go

@@ -90,40 +90,48 @@ func (storage *S3Storage) ListFiles(threadIndex int, dir string) (files []string
 
 	if dir == "snapshots/" {
 		dir = storage.storageDir + dir
-		input := s3.ListObjectsV2Input{
+		input := s3.ListObjectsInput{
 			Bucket:    aws.String(storage.bucket),
 			Prefix:    aws.String(dir),
 			Delimiter: aws.String("/"),
+			MaxKeys:   aws.Int64(1000),
 		}
 
-		err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
-			for _, subDir := range page.CommonPrefixes {
-				files = append(files, (*subDir.Prefix)[len(dir):])
-			}
-			return true
-		})
+		output, err := storage.client.ListObjects(&input)
 		if err != nil {
 			return nil, nil, err
 		}
 
+		for _, subDir := range output.CommonPrefixes {
+			files = append(files, (*subDir.Prefix)[len(dir):])
+		}
 		return files, nil, nil
 	} else {
 		dir = storage.storageDir + dir
-		input := s3.ListObjectsV2Input{
-			Bucket:  aws.String(storage.bucket),
-			Prefix:  aws.String(dir),
-			MaxKeys: aws.Int64(1000),
-		}
+		marker := ""
+		for {
+			input := s3.ListObjectsInput{
+				Bucket:  aws.String(storage.bucket),
+				Prefix:  aws.String(dir),
+				MaxKeys: aws.Int64(1000),
+				Marker:  aws.String(marker),
+			}
 
-		err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
-			for _, object := range page.Contents {
+			output, err := storage.client.ListObjects(&input)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			for _, object := range output.Contents {
 				files = append(files, (*object.Key)[len(dir):])
 				sizes = append(sizes, *object.Size)
 			}
-			return true
-		})
-		if err != nil {
-			return nil, nil, err
+
+			if !*output.IsTruncated {
+				break
+			}
+
+			marker = *output.Contents[len(output.Contents)-1].Key
 		}
 		return files, sizes, nil
 	}

src/duplicacy_storage.go

@@ -756,6 +756,8 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the Storj storage at %s: %v", storageURL, err)
 			return nil
 		}
+		SavePassword(preference, "storj_key", apiKey)
+		SavePassword(preference, "storj_passphrase", passphrase)
 		return storjStorage
 	} else if matched[1] == "smb" {
 		server := matched[3]

src/duplicacy_utils_bsd.go

@@ -1,53 +0,0 @@
-// Copyright (c) Acrosync LLC. All rights reserved.
-// Free for personal use and commercial trial
-// Commercial use requires per-user licenses available from https://duplicacy.com
-
-//go:build freebsd || netbsd || darwin
-// +build freebsd netbsd darwin
-
-package duplicacy
-
-import (
-	"encoding/binary"
-	"os"
-	"syscall"
-)
-
-const bsdFileFlagsKey = "\x00bf"
-
-func (entry *Entry) ReadFileFlags(f *os.File) error {
-	fileInfo, err := f.Stat()
-	if err != nil {
-		return err
-	}
-	stat, ok := fileInfo.Sys().(*syscall.Stat_t)
-	if ok && stat.Flags != 0 {
-		if entry.Attributes == nil {
-			entry.Attributes = &map[string][]byte{}
-		}
-		v := make([]byte, 4)
-		binary.LittleEndian.PutUint32(v, stat.Flags)
-		(*entry.Attributes)[bsdFileFlagsKey] = v
-		LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", stat.Flags, entry.Path)
-	}
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyDirFlags(path string) error {
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
-	return nil
-}
-
-func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
-	if entry.Attributes == nil {
-		return nil
-	}
-	if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
-		LOG_DEBUG("ATTR_RESTORE", "Restore flags 0x%x for %s", binary.LittleEndian.Uint32(v), entry.Path)
-		return syscall.Fchflags(int(f.Fd()), int(binary.LittleEndian.Uint32(v)))
-	}
-	return nil
-}

src/duplicacy_utils_darwin.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 )
 
-func excludedByAttribute(attributes map[string][]byte) bool {
-	value, ok := attributes["com.apple.metadata:com_apple_backup_excludeItem"]
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	value, ok := attirbutes["com.apple.metadata:com_apple_backup_excludeItem"]
 	return ok && strings.Contains(string(value), "com.apple.backupd")
 }

src/duplicacy_utils_freebsd.go

@@ -2,12 +2,12 @@
 // Free for personal use and commercial trial
 // Commercial use requires per-user licenses available from https://duplicacy.com
 
-//go:build freebsd || netbsd || solaris
-// +build freebsd netbsd solaris
-
 package duplicacy
 
-func excludedByAttribute(attributes map[string][]byte) bool {
-	_, ok := attributes["user.duplicacy_exclude"]
+import (
+)
+
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	_, ok := attirbutes["duplicacy_exclude"]
 	return ok
-}
+}

src/duplicacy_utils_linux.go

@@ -5,108 +5,9 @@
 package duplicacy
 
 import (
-	"encoding/binary"
-	"os"
-	"syscall"
-	"unsafe"
 )
 
-const (
-	linux_FS_SECRM_FL        = 0x00000001 /* Secure deletion */
-	linux_FS_UNRM_FL         = 0x00000002 /* Undelete */
-	linux_FS_COMPR_FL        = 0x00000004 /* Compress file */
-	linux_FS_SYNC_FL         = 0x00000008 /* Synchronous updates */
-	linux_FS_IMMUTABLE_FL    = 0x00000010 /* Immutable file */
-	linux_FS_APPEND_FL       = 0x00000020 /* writes to file may only append */
-	linux_FS_NODUMP_FL       = 0x00000040 /* do not dump file */
-	linux_FS_NOATIME_FL      = 0x00000080 /* do not update atime */
-	linux_FS_NOCOMP_FL       = 0x00000400 /* Don't compress */
-	linux_FS_JOURNAL_DATA_FL = 0x00004000 /* Reserved for ext3 */
-	linux_FS_NOTAIL_FL       = 0x00008000 /* file tail should not be merged */
-	linux_FS_DIRSYNC_FL      = 0x00010000 /* dirsync behaviour (directories only) */
-	linux_FS_TOPDIR_FL       = 0x00020000 /* Top of directory hierarchies*/
-	linux_FS_NOCOW_FL        = 0x00800000 /* Do not cow file */
-	linux_FS_PROJINHERIT_FL  = 0x20000000 /* Create with parents projid */
-
-	linux_FS_IOC_GETFLAGS uintptr = 0x80086601
-	linux_FS_IOC_SETFLAGS uintptr = 0x40086602
-
-	linuxIocFlagsFileEarly = linux_FS_SECRM_FL | linux_FS_UNRM_FL | linux_FS_COMPR_FL | linux_FS_NODUMP_FL | linux_FS_NOATIME_FL | linux_FS_NOCOMP_FL | linux_FS_JOURNAL_DATA_FL | linux_FS_NOTAIL_FL | linux_FS_NOCOW_FL
-	linuxIocFlagsDirEarly  = linux_FS_TOPDIR_FL | linux_FS_PROJINHERIT_FL
-	linuxIocFlagsLate      = linux_FS_SYNC_FL | linux_FS_IMMUTABLE_FL | linux_FS_APPEND_FL | linux_FS_DIRSYNC_FL
-
-	linuxFileFlagsKey = "\x00lf"
-)
-
-func ioctl(f *os.File, request uintptr, attrp *uint32) error {
-	argp := uintptr(unsafe.Pointer(attrp))
-
-	if _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), request, argp); errno != 0 {
-		return os.NewSyscallError("ioctl", errno)
-	}
-	return nil
-}
-
-func (entry *Entry) ReadFileFlags(f *os.File) error {
-	var flags uint32
-	if err := ioctl(f, linux_FS_IOC_GETFLAGS, &flags); err != nil {
-		return err
-	}
-	if flags != 0 {
-		if entry.Attributes == nil {
-			entry.Attributes = &map[string][]byte{}
-		}
-		v := make([]byte, 4)
-		binary.LittleEndian.PutUint32(v, flags)
-		(*entry.Attributes)[linuxFileFlagsKey] = v
-		LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", flags, entry.Path)
-	}
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyDirFlags(path string) error {
-	if entry.Attributes == nil {
-		return nil
-	}
-	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
-		flags := binary.LittleEndian.Uint32(v) & linuxIocFlagsDirEarly
-		f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_DIRECTORY, 0)
-		if err != nil {
-			return err
-		}
-		LOG_DEBUG("ATTR_RESTORE", "Restore dir flags (early) 0x%x for %s", flags, entry.Path)
-		err = ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
-		f.Close()
-		return err
-	}
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
-	if entry.Attributes == nil {
-		return nil
-	}
-	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
-		flags := binary.LittleEndian.Uint32(v) & linuxIocFlagsFileEarly
-		LOG_DEBUG("ATTR_RESTORE", "Restore flags (early) 0x%x for %s", flags, entry.Path)
-		return ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
-	}
-	return nil
-}
-
-func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
-	if entry.Attributes == nil {
-		return nil
-	}
-	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
-		flags := binary.LittleEndian.Uint32(v) & (linuxIocFlagsFileEarly | linuxIocFlagsDirEarly | linuxIocFlagsLate)
-		LOG_DEBUG("ATTR_RESTORE", "Restore flags (late) 0x%x for %s", flags, entry.Path)
-		return ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
-	}
-	return nil
-}
-
-func excludedByAttribute(attributes map[string][]byte) bool {
-	_, ok := attributes["user.duplicacy_exclude"]
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	_, ok := attirbutes["duplicacy_exclude"]
 	return ok
 }

src/duplicacy_utils_others.go

@@ -48,12 +48,9 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
 }
 
 func (entry *Entry) ReadAttributes(top string) {
+
 	fullPath := filepath.Join(top, entry.Path)
-	f, err := os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
-	if err != nil {
-		return
-	}
-	attributes, _ := xattr.FList(f)
+	attributes, _ := xattr.List(fullPath)
 	if len(attributes) > 0 {
 		entry.Attributes = &map[string][]byte{}
 		for _, name := range attributes {
@@ -63,42 +60,30 @@ func (entry *Entry) ReadAttributes(top string) {
 			}
 		}
 	}
-	if err := entry.ReadFileFlags(f); err != nil {
-		LOG_INFO("ATTR_BACKUP", "Could not backup flags for file %s: %v", fullPath, err)
-	}
-	f.Close()
 }
 
 func (entry *Entry) SetAttributesToFile(fullPath string) {
-	f, err := os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
-	if err != nil {
-		return
-	}
+	names, _ := xattr.List(fullPath)
 
-	names, _ := xattr.FList(f)
 	for _, name := range names {
+
+
 		newAttribute, found := (*entry.Attributes)[name]
 		if found {
-			oldAttribute, _ := xattr.FGet(f, name)
+			oldAttribute, _ := xattr.Get(fullPath, name)
 			if !bytes.Equal(oldAttribute, newAttribute) {
-				xattr.FSet(f, name, newAttribute)
+				xattr.Set(fullPath, name, newAttribute)
 			}
 			delete(*entry.Attributes, name)
 		} else {
-			xattr.FRemove(f, name)
+			xattr.Remove(fullPath, name)
 		}
 	}
 
 	for name, attribute := range *entry.Attributes {
-		if len(name) > 0 && name[0] == '\x00' {
-			continue
-		}
-		xattr.FSet(f, name, attribute)
+		xattr.Set(fullPath, name, attribute)
 	}
-	if err := entry.RestoreLateFileFlags(f); err != nil {
-		LOG_DEBUG("ATTR_RESTORE", "Could not restore flags for file %s: %v", fullPath, err)
-	}
-	f.Close()
+
 }
 
 func joinPath(components ...string) string {

src/duplicacy_utils_windows.go

@@ -132,18 +132,6 @@ func SplitDir(fullPath string) (dir string, file string) {
 	return fullPath[:i+1], fullPath[i+1:]
 }
 
-func (entry *Entry) ReadFileFlags(f *os.File) error {
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyDirFlags(path string) error {
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
-	return nil
-}
-
-func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
-	return nil
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	return false
 }