Use S3 ListObjectsV2 for listing files

ListObjects has been deprecated since 2016 and ListObjectsV2 with use of
explicit pagination tokens is more performant for large listings as well.

This also mitigates an issue with iDrive E2 where the StartAfter/Marker
is included in the output, leading to duplicate entries. Right now this
causes an exhaustive prune to delete chunks erroneously flagged as
duplicate, destroying the storage.

duplicacy/duplicacy_main.go

@@ -2262,7 +2262,7 @@ func main() {
 	app.Name = "duplicacy"
 	app.HelpName = "duplicacy"
 	app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
-	app.Version = "3.2.2" + " (" + GitCommit + ")"
+	app.Version = "3.2.0" + " (" + GitCommit + ")"
 
 	// Exit with code 2 if an invalid command is provided
 	app.CommandNotFound = func(context *cli.Context, command string) {

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c
 	github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8
 	github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508
+	github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01
 	github.com/hirochachacha/go-smb2 v1.1.0
 	github.com/klauspost/compress v1.16.3
 	github.com/klauspost/reedsolomon v1.9.9

go.sum

@@ -47,6 +47,8 @@ github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8 h1:ijgl4Y+
 github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8/go.mod h1:0lQcVva56+L1PuUFXLOsJ6arJQaU0baIH8q+IegeBhg=
 github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508 h1:SqTyk5KkNXp7zTdTttIZSDcTrL5uau4K/2OpKvgBZVI=
 github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508/go.mod h1:w/pisxUZezf2XzU9Ewjphcf6q1mZtOzKPHhJiuc8cag=
+github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01 h1:LqwS9qL6SrDkp0g0iwUkETrDdtB9gTKaIbSn9imUq5o=
+github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01/go.mod h1:TMlibuxKfkdtHyltooAw7+DHqRpaXs9nxaffk00Sh1Q=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 h1:G1U0vew/vA/1/hBmf1XNeyIzJJbPFVv+kb+HPl6rj6c=

src/duplicacy_entry_test.go

@@ -5,8 +5,6 @@
 package duplicacy
 
 import (
-	"bytes"
-	"encoding/json"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -15,9 +13,10 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"bytes"
+	"encoding/json"
 
-	"github.com/pkg/xattr"
+	"github.com/gilbertchen/xattr"
-
     "github.com/vmihailenco/msgpack"
 )
 
@@ -234,18 +233,8 @@ func TestEntryOrder(t *testing.T) {
 // TestEntryExcludeByAttribute tests the excludeByAttribute parameter to the ListEntries function
 func TestEntryExcludeByAttribute(t *testing.T) {
 
-	var excludeAttrName string
+	if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") {
-	var excludeAttrValue []byte
+		t.Skip("skipping test not darwin or linux")
-
-	if runtime.GOOS == "darwin" {
-		excludeAttrName = "com.apple.metadata:com_apple_backup_excludeItem"
-		excludeAttrValue = []byte("com.apple.backupd")
-	} else if runtime.GOOS == "linux" {
-		excludeAttrName = "user.duplicacy_exclude"
-	} else if runtime.GOOS == "freebsd" || runtime.GOOS == "netbsd" {
-		excludeAttrName = "duplicacy_exclude"
-	} else {
-		t.Skip("skipping test, not darwin, linux, freebsd, or netbsd")
 	}
 
 	testDir := filepath.Join(os.TempDir(), "duplicacy_test")
@@ -284,7 +273,7 @@ func TestEntryExcludeByAttribute(t *testing.T) {
 	for _, file := range DATA {
 		fullPath := filepath.Join(testDir, file)
 		if strings.Contains(file, "exclude") {
-			xattr.Set(fullPath, excludeAttrName, excludeAttrValue)
+			xattr.Setxattr(fullPath, "com.apple.metadata:com_apple_backup_excludeItem", []byte("com.apple.backupd"))
 		}
 	}
 

src/duplicacy_s3storage.go

@@ -90,48 +90,40 @@ func (storage *S3Storage) ListFiles(threadIndex int, dir string) (files []string
 
 	if dir == "snapshots/" {
 		dir = storage.storageDir + dir
-		input := s3.ListObjectsInput{
+		input := s3.ListObjectsV2Input{
 			Bucket:    aws.String(storage.bucket),
 			Prefix:    aws.String(dir),
 			Delimiter: aws.String("/"),
-			MaxKeys:   aws.Int64(1000),
 		}
 
-		output, err := storage.client.ListObjects(&input)
+		err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+			for _, subDir := range page.CommonPrefixes {
+				files = append(files, (*subDir.Prefix)[len(dir):])
+			}
+			return true
+		})
 		if err != nil {
 			return nil, nil, err
 		}
 
-		for _, subDir := range output.CommonPrefixes {
-			files = append(files, (*subDir.Prefix)[len(dir):])
-		}
 		return files, nil, nil
 	} else {
 		dir = storage.storageDir + dir
-		marker := ""
+		input := s3.ListObjectsV2Input{
-		for {
-			input := s3.ListObjectsInput{
 			Bucket:  aws.String(storage.bucket),
 			Prefix:  aws.String(dir),
 			MaxKeys: aws.Int64(1000),
-				Marker:  aws.String(marker),
 		}
 
-			output, err := storage.client.ListObjects(&input)
+		err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
-			if err != nil {
+			for _, object := range page.Contents {
-				return nil, nil, err
-			}
-
-			for _, object := range output.Contents {
 				files = append(files, (*object.Key)[len(dir):])
 				sizes = append(sizes, *object.Size)
 			}
-
+			return true
-			if !*output.IsTruncated {
+		})
-				break
+		if err != nil {
-			}
+			return nil, nil, err
-
-			marker = *output.Contents[len(output.Contents)-1].Key
 		}
 		return files, sizes, nil
 	}

src/duplicacy_storage.go

@@ -756,8 +756,6 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the Storj storage at %s: %v", storageURL, err)
 			return nil
 		}
-		SavePassword(preference, "storj_key", apiKey)
-		SavePassword(preference, "storj_passphrase", passphrase)
 		return storjStorage
 	} else if matched[1] == "smb" {
 		server := matched[3]

src/duplicacy_utils_darwin.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 )
 
-func excludedByAttribute(attributes map[string][]byte) bool {
+func excludedByAttribute(attirbutes map[string][]byte) bool {
-	value, ok := attributes["com.apple.metadata:com_apple_backup_excludeItem"]
+	value, ok := attirbutes["com.apple.metadata:com_apple_backup_excludeItem"]
 	return ok && strings.Contains(string(value), "com.apple.backupd")
 }

src/duplicacy_utils_freebsd.go

@@ -2,12 +2,12 @@
 // Free for personal use and commercial trial
 // Commercial use requires per-user licenses available from https://duplicacy.com
 
-//go:build freebsd || netbsd
-// +build freebsd netbsd
-
 package duplicacy
 
-func excludedByAttribute(attributes map[string][]byte) bool {
+import (
-	_, ok := attributes["duplicacy_exclude"]
+)
+
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	_, ok := attirbutes["duplicacy_exclude"]
 	return ok
 }

src/duplicacy_utils_linux.go

@@ -4,7 +4,10 @@
 
 package duplicacy
 
-func excludedByAttribute(attributes map[string][]byte) bool {
+import (
-	_, ok := attributes["user.duplicacy_exclude"]
+)
+
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	_, ok := attirbutes["duplicacy_exclude"]
 	return ok
 }

src/duplicacy_utils_windows.go

@@ -132,6 +132,6 @@ func SplitDir(fullPath string) (dir string, file string) {
 	return fullPath[:i+1], fullPath[i+1:]
 }
 
-func excludedByAttribute(attributes map[string][]byte) bool {
+func excludedByAttribute(attirbutes map[string][]byte) bool {
 	return false
 }