Initial hardlink in snapshot support.

- Create a new snapshot version number as this method is not backwards
compatible.
- This has some breakages with restoring. Namely if the root file
is not marked for download any hardlinked files that need to be restored
will not be linked, they will be restored as a regular file

duplicacy/duplicacy_main.go

@@ -2262,7 +2262,7 @@ func main() {
 	app.Name = "duplicacy"
 	app.HelpName = "duplicacy"
 	app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
-	app.Version = "3.2.3" + " (" + GitCommit + ")"
+	app.Version = "3.2.0" + " (" + GitCommit + ")"
 
 	// Exit with code 2 if an invalid command is provided
 	app.CommandNotFound = func(context *cli.Context, command string) {

go.mod

@@ -15,7 +15,6 @@ require (
 	github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c
 	github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8
 	github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508
-	github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01
 	github.com/hirochachacha/go-smb2 v1.1.0
 	github.com/klauspost/compress v1.16.3
 	github.com/klauspost/reedsolomon v1.9.9
@@ -29,7 +28,7 @@ require (
 	golang.org/x/net v0.10.0
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	google.golang.org/api v0.21.0
-	storj.io/uplink v1.12.1
+	storj.io/uplink v1.12.0
 )
 
 require (
@@ -56,7 +55,7 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/segmentio/go-env v1.1.0 // indirect
-	github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
+	github.com/spacemonkeygo/monkit/v3 v3.0.20-0.20230227152157-d00b379de191 // indirect
 	github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec // indirect
 	github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
@@ -72,7 +71,7 @@ require (
 	google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d // indirect
 	google.golang.org/grpc v1.28.1 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
-	storj.io/common v0.0.0-20230920095429-0ce0a575e6f8 // indirect
+	storj.io/common v0.0.0-20230907123639-5fd0608fd947 // indirect
 	storj.io/drpc v0.0.33 // indirect
 	storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect
 )

go.sum

@@ -47,8 +47,6 @@ github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8 h1:ijgl4Y+
 github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8/go.mod h1:0lQcVva56+L1PuUFXLOsJ6arJQaU0baIH8q+IegeBhg=
 github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508 h1:SqTyk5KkNXp7zTdTttIZSDcTrL5uau4K/2OpKvgBZVI=
 github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508/go.mod h1:w/pisxUZezf2XzU9Ewjphcf6q1mZtOzKPHhJiuc8cag=
-github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01 h1:LqwS9qL6SrDkp0g0iwUkETrDdtB9gTKaIbSn9imUq5o=
-github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01/go.mod h1:TMlibuxKfkdtHyltooAw7+DHqRpaXs9nxaffk00Sh1Q=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 h1:G1U0vew/vA/1/hBmf1XNeyIzJJbPFVv+kb+HPl6rj6c=
@@ -140,8 +138,8 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/segmentio/go-env v1.1.0 h1:AGJ7OnCx9M5NWpkYPGYELS6III/pFSnAs1GvKWStiEo=
 github.com/segmentio/go-env v1.1.0/go.mod h1:pEKO2ieHe8zF098OMaAHw21SajMuONlnI/vJNB3pB7I=
-github.com/spacemonkeygo/monkit/v3 v3.0.22 h1:4/g8IVItBDKLdVnqrdHZrCVPpIrwDBzl1jrV0IHQHDU=
-github.com/spacemonkeygo/monkit/v3 v3.0.22/go.mod h1:XkZYGzknZwkD0AKUnZaSXhRiVTLCkq7CWVa3IsE72gA=
+github.com/spacemonkeygo/monkit/v3 v3.0.20-0.20230227152157-d00b379de191 h1:QVUfVxilbPp8fBJ7701LL/WEUjBSiSxbs9LUaCIe5qM=
+github.com/spacemonkeygo/monkit/v3 v3.0.20-0.20230227152157-d00b379de191/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
@@ -195,6 +193,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -292,11 +291,11 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
-storj.io/common v0.0.0-20230920095429-0ce0a575e6f8 h1:i+bWPhVnNL6z/TLW3vDZytB6/0bsvJM0a1GhLCxrlxQ=
-storj.io/common v0.0.0-20230920095429-0ce0a575e6f8/go.mod h1:ZmeGPzRb2sm705Nwt/WwuH3e6mliShfvvoUNy1bb9v4=
+storj.io/common v0.0.0-20230907123639-5fd0608fd947 h1:X75A5hX1nFjQH8GIvei4T1LNQTLa++bsDKMxXxfPHE8=
+storj.io/common v0.0.0-20230907123639-5fd0608fd947/go.mod h1:FMVOxf2+SgsmfjxwFCM1MZCKwXis4U7l22M/6nIhIas=
 storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI=
 storj.io/drpc v0.0.33/go.mod h1:vR804UNzhBa49NOJ6HeLjd2H3MakC1j5Gv8bsOQT6N4=
 storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c h1:or/DtG5uaZpzimL61ahlgAA+MTYn/U3txz4fe+XBFUg=
 storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c/go.mod h1:JCuc3C0gzCJHQ4J6SOx/Yjg+QTpX0D+Fvs5H46FETCk=
-storj.io/uplink v1.12.1 h1:bDc2dI6Q7EXcvPJLZuH9jIOTIf2oKxvW3xKEA+Y5EI0=
-storj.io/uplink v1.12.1/go.mod h1:1+czctHG25pMzcUp4Mds6QnoJ7LvbgYA5d1qlpFFexg=
+storj.io/uplink v1.12.0 h1:rTODjbKRo/lzz5Hp0isjoRfqDcH7kJg6aujD2M9v9Ro=
+storj.io/uplink v1.12.0/go.mod h1:nMAuoWi5AHio+8NQa33VRzCiRg0B0UhYKuT0a0CdXOg=

src/duplicacy_backupmanager.go

@@ -622,6 +622,11 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 	return true
 }
 
+type hardLinkEntry struct {
+	entry *Entry
+	willDownload bool
+}
+
 // Restore downloads the specified snapshot, compares it with what's on the repository, and then downloads
 // files that are different. 'base' is a directory that contains files at a different revision which can
 // serve as a local cache to avoid download chunks available locally.  It is perfectly ok for 'base' to be
@@ -703,8 +708,16 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 	var localEntry *Entry
 	localListingOK := true
 
+	hardLinkTable := make(map[string]hardLinkEntry)
+	hardLinks := make([]*Entry, 0)
+
 	for remoteEntry := range remoteListingChannel {
 
+		if remoteEntry.IsFile() && remoteEntry.Link == "/" {
+			LOG_INFO("RESTORE_LINK", "Noting hardlinked source file %s", remoteEntry.Path)
+			hardLinkTable[remoteEntry.Path] = hardLinkEntry{remoteEntry, false}
+		}
+
 		if len(patterns) > 0 && !MatchPath(remoteEntry.Path, patterns) {
 			continue
 		}
@@ -713,6 +726,8 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		var compareResult int
 
 		for {
+			// TODO: We likely need to check if a local listing file exists in the hardLinkTable for the case where one is restoring a hardlink
+			// to an existing disk file. Right now, we'll just end up downloading the file new.
 			if localEntry == nil && localListingOK {
 				localEntry, localListingOK = <- localListingChannel
 			}
@@ -730,12 +745,22 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		}
 
 		if compareResult == 0 {
-			if quickMode && localEntry.IsFile() && localEntry.IsSameAs(remoteEntry) {
-				LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", localEntry.Path)
-				skippedFileSize += localEntry.Size
-				skippedFileCount++
-				localEntry = nil
-				continue
+			if quickMode && localEntry.IsFile() {
+				checkEntry := remoteEntry
+				if len(remoteEntry.Link) > 0 && remoteEntry.Link != "/" {
+					if e, ok := hardLinkTable[remoteEntry.Link]; !ok {
+						LOG_ERROR("RESTORE_LINK", "Source file %s for hardlink %s missing", remoteEntry.Link, remoteEntry.Path)
+					} else {
+						checkEntry = e.entry
+					}
+				}
+				if localEntry.IsSameAs(checkEntry) {
+					LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", localEntry.Path)
+					skippedFileSize += localEntry.Size
+					skippedFileCount++
+					localEntry = nil
+					continue
+				}
 			}
 			localEntry = nil
 		}
@@ -782,6 +807,21 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 			}
 			directoryEntries = append(directoryEntries, remoteEntry)
 		} else {
+			if remoteEntry.Link == "/" {
+				hardLinkTable[remoteEntry.Path] = hardLinkEntry{remoteEntry, true}
+			} else if len(remoteEntry.Link) > 0 {
+				if e, ok := hardLinkTable[remoteEntry.Link]; !ok {
+					LOG_ERROR("RESTORE_LINK", "Source file %s for hardlink %s missing", remoteEntry.Link, remoteEntry.Path)
+				} else if !e.willDownload {
+					origSourcePath := e.entry.Path
+					e.entry.Path = remoteEntry.Path
+					remoteEntry = e.entry
+					hardLinkTable[origSourcePath] = hardLinkEntry{remoteEntry, true}
+				} else {
+					hardLinks = append(hardLinks, remoteEntry)
+					continue
+				}
+			}
 			// We can't download files here since fileEntries needs to be sorted
 			fileEntries = append(fileEntries, remoteEntry)
 			totalFileSize += remoteEntry.Size
@@ -837,7 +877,11 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		stat, _ := os.Stat(fullPath)
 		if stat != nil {
 			if quickMode {
-				if file.IsSameAsFileInfo(stat) {
+				cmpFile := file
+				if file.IsFile() && len(file.Link) > 0 && file.Link != "/" {
+					cmpFile = hardLinkTable[file.Link].entry
+				}
+				if cmpFile.IsSameAsFileInfo(stat) {
 					LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", file.Path)
 					skippedFileSize += file.Size
 					skippedFileCount++
@@ -874,7 +918,6 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 				downloadedFileSize += file.Size
 				downloadedFiles = append(downloadedFiles, file)
 			}
-
 			continue
 		}
 
@@ -901,6 +944,15 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		file.RestoreMetadata(fullPath, nil, setOwner)
 	}
 
+	for _, linkEntry := range hardLinks {
+		sourcePath := joinPath(top, hardLinkTable[linkEntry.Link].entry.Path)
+		fullPath := joinPath(top, linkEntry.Path)
+		LOG_INFO("DOWNLOAD_LINK", "Hard linking %s -> %s", fullPath, sourcePath)
+		if err := os.Link(sourcePath, fullPath); err != nil {
+			LOG_ERROR("DOWNLOAD_LINK", "Failed to create hard link %s -> %s", fullPath, sourcePath)
+		}
+	}
+
 	if deleteMode && len(patterns) == 0 {
 		// Reverse the order to make sure directories are empty before being deleted
 		for i := range extraFiles {

src/duplicacy_entry.go

@@ -18,6 +18,7 @@ import (
 	"time"
 	"bytes"
 	"crypto/sha256"
+	"syscall"
 
     "github.com/vmihailenco/msgpack"
 
@@ -694,10 +695,26 @@ func (files FileInfoCompare) Less(i, j int) bool {
 	}
 }
 
+type listEntryLinkKey struct {
+	dev uint64
+	ino uint64
+}
+
+type ListingState struct {
+	linkTable map[listEntryLinkKey]string // map unique inode details to initially found path
+}
+
+func NewListingState() *ListingState {
+	return &ListingState{
+		linkTable: make(map[listEntryLinkKey]string),
+	}
+}
+
 // ListEntries returns a list of entries representing file and subdirectories under the directory 'path'.  Entry paths
 // are normalized as relative to 'top'.  'patterns' are used to exclude or include certain files.
-func ListEntries(top string, path string, patterns []string, nobackupFile string, excludeByAttribute bool, listingChannel chan *Entry) (directoryList []*Entry,
-	skippedFiles []string, err error) {
+func ListEntries(top string, path string, patterns []string, nobackupFile string, excludeByAttribute bool,
+	listingState *ListingState,
+	listingChannel chan *Entry) (directoryList []*Entry, skippedFiles []string, err error) {
 
 	LOG_DEBUG("LIST_ENTRIES", "Listing %s", path)
 
@@ -784,6 +801,21 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
 			continue
 		}
 
+		if entry.IsFile() {
+			stat, ok := f.Sys().(*syscall.Stat_t)
+			if ok && stat != nil && stat.Nlink > 1 {
+				k := listEntryLinkKey{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}
+				if path, ok := listingState.linkTable[k]; ok {
+					LOG_WARN("LIST_LINK", "Linking %s to %s", entry.Path, path)
+					entry.Link = path
+					entry.Size = 0
+				} else {
+					entry.Link = "/"
+					listingState.linkTable[k] = entry.Path
+				}
+			}
+		}
+
 		if entry.IsDir() {
 			directoryList = append(directoryList, entry)
 		} else {

src/duplicacy_entry_test.go

@@ -5,6 +5,8 @@
 package duplicacy
 
 import (
+	"bytes"
+	"encoding/json"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -13,11 +15,10 @@ import (
 	"sort"
 	"strings"
 	"testing"
-	"bytes"
-	"encoding/json"
 
-	"github.com/gilbertchen/xattr"
-    "github.com/vmihailenco/msgpack"
+	"github.com/pkg/xattr"
+
+	"github.com/vmihailenco/msgpack"
 )
 
 func TestEntrySort(t *testing.T) {
@@ -175,7 +176,7 @@ func TestEntryOrder(t *testing.T) {
 	directories = append(directories, CreateEntry("", 0, 0, 0))
 
 	entries := make([]*Entry, 0, 4)
-    entryChannel := make(chan *Entry, 1024)
+	entryChannel := make(chan *Entry, 1024)
 	entries = append(entries, CreateEntry("", 0, 0, 0))
 
 	for len(directories) > 0 {
@@ -233,8 +234,16 @@ func TestEntryOrder(t *testing.T) {
 // TestEntryExcludeByAttribute tests the excludeByAttribute parameter to the ListEntries function
 func TestEntryExcludeByAttribute(t *testing.T) {
 
-	if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") {
-		t.Skip("skipping test not darwin or linux")
+	var excludeAttrName string
+	var excludeAttrValue []byte
+
+	if runtime.GOOS == "darwin" {
+		excludeAttrName = "com.apple.metadata:com_apple_backup_excludeItem"
+		excludeAttrValue = []byte("com.apple.backupd")
+	} else if runtime.GOOS == "linux" || runtime.GOOS == "freebsd" || runtime.GOOS == "netbsd" || runtime.GOOS == "solaris" {
+		excludeAttrName = "user.duplicacy_exclude"
+	} else {
+		t.Skip("skipping test, not darwin, linux, freebsd, netbsd, or solaris")
 	}
 
 	testDir := filepath.Join(os.TempDir(), "duplicacy_test")
@@ -273,7 +282,7 @@ func TestEntryExcludeByAttribute(t *testing.T) {
 	for _, file := range DATA {
 		fullPath := filepath.Join(testDir, file)
 		if strings.Contains(file, "exclude") {
-			xattr.Setxattr(fullPath, "com.apple.metadata:com_apple_backup_excludeItem", []byte("com.apple.backupd"))
+			xattr.Set(fullPath, excludeAttrName, excludeAttrValue)
 		}
 	}
 
@@ -372,4 +381,4 @@ func TestEntryEncoding(t *testing.T) {
 		t.Error("Decoded entry is different than the original one")
 	}
 
-}
+}

src/duplicacy_s3storage.go

@@ -90,48 +90,40 @@ func (storage *S3Storage) ListFiles(threadIndex int, dir string) (files []string
 
 	if dir == "snapshots/" {
 		dir = storage.storageDir + dir
-		input := s3.ListObjectsInput{
+		input := s3.ListObjectsV2Input{
 			Bucket:    aws.String(storage.bucket),
 			Prefix:    aws.String(dir),
 			Delimiter: aws.String("/"),
-			MaxKeys:   aws.Int64(1000),
 		}
 
-		output, err := storage.client.ListObjects(&input)
+		err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+			for _, subDir := range page.CommonPrefixes {
+				files = append(files, (*subDir.Prefix)[len(dir):])
+			}
+			return true
+		})
 		if err != nil {
 			return nil, nil, err
 		}
 
-		for _, subDir := range output.CommonPrefixes {
-			files = append(files, (*subDir.Prefix)[len(dir):])
-		}
 		return files, nil, nil
 	} else {
 		dir = storage.storageDir + dir
-		marker := ""
-		for {
-			input := s3.ListObjectsInput{
-				Bucket:  aws.String(storage.bucket),
-				Prefix:  aws.String(dir),
-				MaxKeys: aws.Int64(1000),
-				Marker:  aws.String(marker),
-			}
+		input := s3.ListObjectsV2Input{
+			Bucket:  aws.String(storage.bucket),
+			Prefix:  aws.String(dir),
+			MaxKeys: aws.Int64(1000),
+		}
 
-			output, err := storage.client.ListObjects(&input)
-			if err != nil {
-				return nil, nil, err
-			}
-
-			for _, object := range output.Contents {
+		err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+			for _, object := range page.Contents {
 				files = append(files, (*object.Key)[len(dir):])
 				sizes = append(sizes, *object.Size)
 			}
-
-			if !*output.IsTruncated {
-				break
-			}
-
-			marker = *output.Contents[len(output.Contents)-1].Key
+			return true
+		})
+		if err != nil {
+			return nil, nil, err
 		}
 		return files, sizes, nil
 	}

src/duplicacy_snapshot.go

@@ -51,7 +51,7 @@ type Snapshot struct {
 // CreateEmptySnapshot creates an empty snapshot.
 func CreateEmptySnapshot(id string) (snapshto *Snapshot) {
 	return &Snapshot{
-		Version:   1,
+		Version:   0x6a6c01,
 		ID:        id,
 		Revision:  0,
 		StartTime: time.Now().Unix(),
@@ -68,6 +68,7 @@ func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
 								         skippedDirectories *[]string, skippedFiles *[]string) {
 
 	var patterns []string
+	var listingState = NewListingState()
 
 	if filtersFile == "" {
 		filtersFile = joinPath(GetDuplicacyPreferencePath(), "filters")
@@ -81,7 +82,7 @@ func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
 
 		directory := directories[len(directories)-1]
 		directories = directories[:len(directories)-1]
-		subdirectories, skipped, err := ListEntries(top, directory.Path, patterns, nobackupFile, excludeByAttribute, listingChannel)
+		subdirectories, skipped, err := ListEntries(top, directory.Path, patterns, nobackupFile, excludeByAttribute, listingState, listingChannel)
 		if err != nil {
 			if directory.Path == "" {
 				LOG_ERROR("LIST_FAILURE", "Failed to list the repository root: %v", err)
@@ -160,7 +161,7 @@ func (snapshot *Snapshot)ListRemoteFiles(config *Config, chunkOperator *ChunkOpe
 				return
 			}
 		}
-	} else if snapshot.Version == 1 {
+	} else if snapshot.Version == 1 || snapshot.Version == 0x6a6c01 {
 		decoder := msgpack.NewDecoder(reader)
 
 		lastEndChunk := 0

src/duplicacy_storage.go

@@ -756,8 +756,6 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the Storj storage at %s: %v", storageURL, err)
 			return nil
 		}
-		SavePassword(preference, "storj_key", apiKey)
-		SavePassword(preference, "storj_passphrase", passphrase)
 		return storjStorage
 	} else if matched[1] == "smb" {
 		server := matched[3]

src/duplicacy_utils_darwin.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 )
 
-func excludedByAttribute(attirbutes map[string][]byte) bool {
-	value, ok := attirbutes["com.apple.metadata:com_apple_backup_excludeItem"]
+func excludedByAttribute(attributes map[string][]byte) bool {
+	value, ok := attributes["com.apple.metadata:com_apple_backup_excludeItem"]
 	return ok && strings.Contains(string(value), "com.apple.backupd")
 }

src/duplicacy_utils_freebsd.go

@@ -1,13 +0,0 @@
-// Copyright (c) Acrosync LLC. All rights reserved.
-// Free for personal use and commercial trial
-// Commercial use requires per-user licenses available from https://duplicacy.com
-
-package duplicacy
-
-import (
-)
-
-func excludedByAttribute(attirbutes map[string][]byte) bool {
-	_, ok := attirbutes["duplicacy_exclude"]
-	return ok
-}

src/duplicacy_utils_posix.go

@@ -2,12 +2,14 @@
 // Free for personal use and commercial trial
 // Commercial use requires per-user licenses available from https://duplicacy.com
 
+// +build freebsd netbsd linux solaris
+
 package duplicacy
 
 import (
 )
 
-func excludedByAttribute(attirbutes map[string][]byte) bool {
-	_, ok := attirbutes["duplicacy_exclude"]
+func excludedByAttribute(attributes map[string][]byte) bool {
+	_, ok := attributes["user.duplicacy_exclude"]
 	return ok
 }

src/duplicacy_utils_windows.go

@@ -132,6 +132,6 @@ func SplitDir(fullPath string) (dir string, file string) {
 	return fullPath[:i+1], fullPath[i+1:]
 }
 
-func excludedByAttribute(attirbutes map[string][]byte) bool {
+func excludedByAttribute(attributes map[string][]byte) bool {
 	return false
 }