Support backup of sockets

This only works on Linux. Darwin does not allow mknod of a socket.
Have not tested BSD.

go.mod

@@ -15,7 +15,6 @@ require (
 	github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c
 	github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8
 	github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508
-	github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01
 	github.com/hirochachacha/go-smb2 v1.1.0
 	github.com/klauspost/compress v1.16.3
 	github.com/klauspost/reedsolomon v1.9.9

go.sum

@@ -47,8 +47,6 @@ github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8 h1:ijgl4Y+
 github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8/go.mod h1:0lQcVva56+L1PuUFXLOsJ6arJQaU0baIH8q+IegeBhg=
 github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508 h1:SqTyk5KkNXp7zTdTttIZSDcTrL5uau4K/2OpKvgBZVI=
 github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508/go.mod h1:w/pisxUZezf2XzU9Ewjphcf6q1mZtOzKPHhJiuc8cag=
-github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01 h1:LqwS9qL6SrDkp0g0iwUkETrDdtB9gTKaIbSn9imUq5o=
-github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01/go.mod h1:TMlibuxKfkdtHyltooAw7+DHqRpaXs9nxaffk00Sh1Q=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 h1:G1U0vew/vA/1/hBmf1XNeyIzJJbPFVv+kb+HPl6rj6c=

src/duplicacy_b2client.go

@@ -396,7 +396,7 @@ type B2ListFileNamesOutput struct {
 
 func (client *B2Client) ListFileNames(threadIndex int, startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
 
-	maxFileCount := 1000
+	maxFileCount := 10_000
 	if singleFile {
 		if includeVersions {
 			maxFileCount = 4

src/duplicacy_backupmanager.go

@@ -21,7 +21,7 @@ import (
 	"sync/atomic"
 	"time"
 
-    "github.com/vmihailenco/msgpack"
+	"github.com/vmihailenco/msgpack"
 )
 
 // BackupManager performs the two major operations, backup and restore, and passes other operations, mostly related to
@@ -36,9 +36,9 @@ type BackupManager struct {
 
 	config *Config // contains a number of options
 
-	nobackupFile string // don't backup directory when this file name is found
+	nobackupFile       string // don't backup directory when this file name is found
-	filtersFile string  // the path to the filters file
+	filtersFile        string // the path to the filters file
-	excludeByAttribute bool // don't backup file based on file attribute
+	excludeByAttribute bool   // don't backup file based on file attribute
 
 	cachePath string
 }
@@ -146,7 +146,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 
 	if manager.config.DataShards != 0 && manager.config.ParityShards != 0 {
 		LOG_INFO("BACKUP_ERASURECODING", "Erasure coding is enabled with %d data shards and %d parity shards",
-		         manager.config.DataShards, manager.config.ParityShards)
+			manager.config.DataShards, manager.config.ParityShards)
 	}
 
 	if manager.config.rsaPublicKey != nil && len(manager.config.FileKey) > 0 {
@@ -187,7 +187,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 
 	// If the listing operation is fast and this is an initial backup, list all chunks and
 	// put them in the cache.
-	if (manager.storage.IsFastListing() && remoteSnapshot.Revision == 0) {
+	if manager.storage.IsFastListing() && remoteSnapshot.Revision == 0 {
 		LOG_INFO("BACKUP_LIST", "Listing all chunks")
 		allChunks, _ := manager.SnapshotManager.ListAllFiles(manager.storage, "chunks/")
 
@@ -222,7 +222,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 
 	var totalModifiedFileSize int64    // total size of modified files
 	var uploadedModifiedFileSize int64 // portions that have been uploaded (including cache hits)
-	var preservedFileSize int64 // total size of unmodified files
+	var preservedFileSize int64        // total size of unmodified files
 
 	localSnapshot := CreateEmptySnapshot(manager.snapshotID)
 	localSnapshot.Revision = remoteSnapshot.Revision + 1
@@ -239,7 +239,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 		// List local files
 		defer CatchLogException()
 		localSnapshot.ListLocalFiles(shadowTop, manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute, localListingChannel, &skippedDirectories, &skippedFiles)
-	} ()
+	}()
 
 	go func() {
 		// List remote files
@@ -261,7 +261,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 			})
 		}
 		close(remoteListingChannel)
-	} ()
+	}()
 
 	// Create the local file list
 	localEntryList, err := CreateEntryList(manager.snapshotID, manager.cachePath, maximumInMemoryEntries)
@@ -275,7 +275,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 	var remoteEntry *Entry
 	remoteListingOK := true
 	for {
-		localEntry := <- localListingChannel
+		localEntry := <-localListingChannel
 		if localEntry == nil {
 			break
 		}
@@ -289,7 +289,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 				compareResult = localEntry.Compare(remoteEntry)
 			} else {
 				if remoteListingOK {
-					remoteEntry, remoteListingOK = <- remoteListingChannel
+					remoteEntry, remoteListingOK = <-remoteListingChannel
 				}
 				if !remoteListingOK {
 					compareResult = -1
@@ -304,7 +304,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 			remoteEntry = nil
 		}
 
-		if compareResult == 0  {
+		if compareResult == 0 {
 			// No need to check if it is in hash mode -- in that case remote listing is nil
 			if localEntry.IsSameAs(remoteEntry) && localEntry.IsFile() {
 				if localEntry.Size > 0 {
@@ -339,8 +339,8 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 			// compareResult must be < 0; the local file is new
 			totalModifiedFileSize += localEntry.Size
 			if localEntry.Size > 0 {
-					// A size of -1 indicates this is a modified file that will be uploaded
+				// A size of -1 indicates this is a modified file that will be uploaded
-					localEntry.Size = -1
+				localEntry.Size = -1
 			}
 		}
 
@@ -448,7 +448,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 
 		_, found := chunkCache[chunkID]
 		if found {
-			if time.Now().Unix() - lastUploadingTime > keepUploadAlive {
+			if time.Now().Unix()-lastUploadingTime > keepUploadAlive {
 				LOG_INFO("UPLOAD_KEEPALIVE", "Skip chunk cache to keep connection alive")
 				found = false
 			}
@@ -558,7 +558,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 	if showStatistics {
 
 		LOG_INFO("BACKUP_STATS", "Files: %d total, %s bytes; %d new, %s bytes",
-			localEntryList.NumberOfEntries - int64(len(skippedFiles)),
+			localEntryList.NumberOfEntries-int64(len(skippedFiles)),
 			PrettyNumber(preservedFileSize+uploadedFileSize),
 			len(localEntryList.ModifiedEntries), PrettyNumber(uploadedFileSize))
 
@@ -686,7 +686,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		// List local files
 		defer CatchLogException()
 		localSnapshot.ListLocalFiles(top, manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute, localListingChannel, nil, nil)
-	} ()
+	}()
 
 	remoteSnapshot := manager.SnapshotManager.DownloadSnapshot(manager.snapshotID, revision)
 	manager.SnapshotManager.DownloadSnapshotSequences(remoteSnapshot)
@@ -698,13 +698,47 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 			return true
 		})
 		close(remoteListingChannel)
-	} ()
+	}()
 
 	var localEntry *Entry
 	localListingOK := true
 
+	type hardLinkEntry struct {
+		entry     *Entry
+		willExist bool
+	}
+	var hardLinkTable []hardLinkEntry
+	var hardLinks []*Entry
+
+	restoreHardlink := func(entry *Entry, fullPath string) bool {
+		if entry.IsHardlinkRoot() {
+			hardLinkTable[len(hardLinkTable)-1].willExist = true
+		} else if entry.IsHardlinkedFrom() {
+			i, err := entry.GetHardlinkId()
+			if err != nil {
+				LOG_ERROR("RESTORE_HARDLINK", "Decode error for hardlinked entry %s, %v", entry.Path, err)
+				return false
+			}
+			if !hardLinkTable[i].willExist {
+				hardLinkTable[i] = hardLinkEntry{entry, true}
+			} else {
+				sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
+				LOG_INFO("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath)
+				if err := MakeHardlink(sourcePath, fullPath); err != nil {
+					LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s %v", fullPath, sourcePath, err)
+				}
+				return true
+			}
+		}
+		return false
+	}
+
 	for remoteEntry := range remoteListingChannel {
 
+		if remoteEntry.IsHardlinkRoot() {
+			hardLinkTable = append(hardLinkTable, hardLinkEntry{remoteEntry, false})
+		}
+
 		if len(patterns) > 0 && !MatchPath(remoteEntry.Path, patterns) {
 			continue
 		}
@@ -714,7 +748,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 
 		for {
 			if localEntry == nil && localListingOK {
-				localEntry, localListingOK = <- localListingChannel
+				localEntry, localListingOK = <-localListingChannel
 			}
 			if localEntry == nil {
 				compareResult = 1
@@ -741,26 +775,39 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		}
 
 		fullPath := joinPath(top, remoteEntry.Path)
+
 		if remoteEntry.IsLink() {
-			stat, err := os.Lstat(fullPath)
+			if stat, _ := os.Lstat(fullPath); stat != nil {
-			if stat != nil {
 				if stat.Mode()&os.ModeSymlink != 0 {
 					isRegular, link, err := Readlink(fullPath)
 					if err == nil && link == remoteEntry.Link && !isRegular {
 						remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
+						if remoteEntry.IsHardlinkRoot() {
+							hardLinkTable[len(hardLinkTable)-1].willExist = true
+						}
 						continue
 					}
 				}
 
+				if !overwrite {
+					LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
+						"File %s already exists.  Please specify the -overwrite option to overwrite", remoteEntry.Path)
+					continue
+				}
+
 				os.Remove(fullPath)
 			}
 
-			err = os.Symlink(remoteEntry.Link, fullPath)
+			if restoreHardlink(remoteEntry, fullPath) {
-			if err != nil {
+				continue
+			}
+
+			if err := os.Symlink(remoteEntry.Link, fullPath); err != nil {
 				LOG_ERROR("RESTORE_SYMLINK", "Can't create symlink %s: %v", remoteEntry.Path, err)
 				return 0
 			}
 			remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
+
 			LOG_TRACE("DOWNLOAD_DONE", "Symlink %s updated", remoteEntry.Path)
 		} else if remoteEntry.IsDir() {
 
@@ -780,8 +827,50 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 					return 0
 				}
 			}
+			remoteEntry.RestoreEarlyDirFlags(fullPath)
 			directoryEntries = append(directoryEntries, remoteEntry)
+		} else if remoteEntry.IsSpecial() {
+			if stat, _ := os.Lstat(fullPath); stat != nil {
+				if remoteEntry.IsSameSpecial(stat) {
+					remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
+					if remoteEntry.IsHardlinkRoot() {
+						hardLinkTable[len(hardLinkTable)-1].willExist = true
+					}
+				}
+				if !overwrite {
+					LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
+						"File %s already exists.  Please specify the -overwrite option to overwrite", remoteEntry.Path)
+					continue
+				}
+				os.Remove(fullPath)
+			}
+
+			if restoreHardlink(remoteEntry, fullPath) {
+				continue
+			}
+
+			if err := remoteEntry.RestoreSpecial(fullPath); err != nil {
+				LOG_ERROR("RESTORE_SPECIAL", "Unable to restore special file %s: %v", remoteEntry.Path, err)
+				return 0
+			}
+			remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
 		} else {
+			if remoteEntry.IsHardlinkRoot() {
+				hardLinkTable[len(hardLinkTable)-1].willExist = true
+			} else if remoteEntry.IsHardlinkedFrom() {
+				i, err := remoteEntry.GetHardlinkId()
+				if err != nil {
+					LOG_ERROR("RESTORE_HARDLINK", "Decode error for hardlinked entry %s, %v", remoteEntry.Path, err)
+					return 0
+				}
+				if !hardLinkTable[i].willExist {
+					hardLinkTable[i] = hardLinkEntry{remoteEntry, true}
+				} else {
+					hardLinks = append(hardLinks, remoteEntry)
+					continue
+				}
+			}
+
 			// We can't download files here since fileEntries needs to be sorted
 			fileEntries = append(fileEntries, remoteEntry)
 			totalFileSize += remoteEntry.Size
@@ -793,7 +882,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 	}
 
 	for localListingOK {
-		localEntry, localListingOK = <- localListingChannel
+		localEntry, localListingOK = <-localListingChannel
 		if localEntry != nil {
 			extraFiles = append(extraFiles, localEntry.Path)
 		}
@@ -901,6 +990,38 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		file.RestoreMetadata(fullPath, nil, setOwner)
 	}
 
+	for _, linkEntry := range hardLinks {
+
+		i, _ := linkEntry.GetHardlinkId()
+		sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
+		fullPath := joinPath(top, linkEntry.Path)
+
+		if stat, _ := os.Lstat(fullPath); stat != nil {
+			sourceStat, _ := os.Lstat(sourcePath)
+			if os.SameFile(stat, sourceStat) {
+				continue
+			}
+
+			if sourceStat == nil {
+				LOG_WERROR(allowFailures, "RESTORE_HARDLINK",
+					"Target %s for hardlink %s is missing", sourcePath, linkEntry.Path)
+				continue
+			}
+			if !overwrite {
+				LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
+					"File %s already exists.  Please specify the -overwrite option to overwrite", linkEntry.Path)
+				continue
+			}
+			os.Remove(fullPath)
+		}
+
+		LOG_DEBUG("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath)
+		if err := MakeHardlink(sourcePath, fullPath); err != nil {
+			LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s", fullPath, sourcePath)
+			return 0
+		}
+	}
+
 	if deleteMode && len(patterns) == 0 {
 		// Reverse the order to make sure directories are empty before being deleted
 		for i := range extraFiles {
@@ -1046,14 +1167,19 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
 	encoder := msgpack.NewEncoder(buffer)
 	metadataChunkMaker := CreateMetaDataChunkMaker(manager.config, metadataChunkSize)
 
-    var chunkHashes  []string
+	var chunkHashes []string
 	var chunkLengths []int
 	lastChunk := -1
 
 	lastEndChunk := 0
 
-	uploadEntryInfoFunc := func(entry *Entry) error {
+	type hardLinkEntry struct {
+		entry      *Entry
+		startChunk int
+	}
+	var hardLinkTable []hardLinkEntry
 
+	uploadEntryInfoFunc := func(entry *Entry) error {
 		if entry.IsFile() && entry.Size > 0 {
 			delta := entry.StartChunk - len(chunkHashes) + 1
 			if entry.StartChunk != lastChunk {
@@ -1071,10 +1197,38 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
 			entry.StartChunk -= delta
 			entry.EndChunk -= delta
 
+			if entry.IsHardlinkRoot() {
+				LOG_DEBUG("SNAPSHOT_UPLOAD", "Hard link root %s %v %v", entry.Path, entry.StartChunk, entry.EndChunk)
+				hardLinkTable = append(hardLinkTable, hardLinkEntry{entry, entry.StartChunk})
+			}
+
 			delta = entry.EndChunk - entry.StartChunk
 			entry.StartChunk -= lastEndChunk
 			lastEndChunk = entry.EndChunk
 			entry.EndChunk = delta
+		} else if entry.IsHardlinkedFrom() && !entry.IsLink() {
+			i, err := entry.GetHardlinkId()
+			if err != nil {
+				LOG_ERROR("SNAPSHOT_UPLOAD", "Decode error for hardlinked entry %s, %v", entry.Link, err)
+				return err
+			}
+
+			targetEntry := hardLinkTable[i].entry
+			var startChunk, endChunk int
+
+			if targetEntry.Size > 0 {
+				startChunk = hardLinkTable[i].startChunk - lastEndChunk
+				endChunk = targetEntry.EndChunk
+			}
+			entry = entry.HardLinkTo(targetEntry, startChunk, endChunk)
+
+			if targetEntry.Size > 0 {
+				lastEndChunk = hardLinkTable[i].startChunk + endChunk
+			}
+
+			LOG_DEBUG("SNAPSHOT_UPLOAD", "Uploading cloned hardlink for %s to %s (%v %v)", entry.Path, targetEntry.Path, startChunk, endChunk)
+		} else if entry.IsHardlinkRoot() {
+			hardLinkTable = append(hardLinkTable, hardLinkEntry{entry, 0})
 		}
 
 		buffer.Reset()
@@ -1143,9 +1297,10 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
 // Restore downloads a file from the storage.  If 'inPlace' is false, the download file is saved first to a temporary
 // file under the .duplicacy directory and then replaces the existing one.  Otherwise, the existing file will be
 // overwritten directly.
-// Return: true, nil:    Restored file; 
+// Return: true, nil:    Restored file;
-//         false, nil:   Skipped file; 
+//
-//         false, error: Failure to restore file (only if allowFailures == true)
+//	false, nil:   Skipped file;
+//	false, error: Failure to restore file (only if allowFailures == true)
 func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chunkMaker *ChunkMaker, entry *Entry, top string, inPlace bool, overwrite bool,
 	showStatistics bool, totalFileSize int64, downloadedFileSize int64, startTime int64, allowFailures bool) (bool, error) {
 
@@ -1194,6 +1349,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 					LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing: %v", fullPath, err)
 					return false, nil
 				}
+				entry.RestoreEarlyFileFlags(existingFile)
 
 				n := int64(1)
 				// There is a go bug on Windows (https://github.com/golang/go/issues/21681) that causes Seek to fail
@@ -1320,7 +1476,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			// fileHash != entry.Hash, warn/error depending on -overwrite option
 			if !overwrite && !isNewFile {
 				LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
-							"File %s already exists.  Please specify the -overwrite option to overwrite", entry.Path)
+					"File %s already exists.  Please specify the -overwrite option to overwrite", entry.Path)
 				return false, fmt.Errorf("file exists")
 			}
 
@@ -1377,6 +1533,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 				return false, nil
 			}
 		}
+		entry.RestoreEarlyFileFlags(existingFile)
 
 		existingFile.Seek(0, 0)
 
@@ -1459,6 +1616,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			LOG_ERROR("DOWNLOAD_OPEN", "Failed to open file for writing: %v", err)
 			return false, nil
 		}
+		entry.RestoreEarlyFileFlags(newFile)
 
 		hasher := manager.config.NewFileHasher()
 
@@ -1565,7 +1723,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 
 	if otherManager.config.DataShards != 0 && otherManager.config.ParityShards != 0 {
 		LOG_INFO("BACKUP_ERASURECODING", "Erasure coding is enabled for the destination storage with %d data shards and %d parity shards",
-		         otherManager.config.DataShards, otherManager.config.ParityShards)
+			otherManager.config.DataShards, otherManager.config.ParityShards)
 	}
 
 	if otherManager.config.rsaPublicKey != nil && len(otherManager.config.FileKey) > 0 {
@@ -1666,15 +1824,15 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 		LOG_TRACE("SNAPSHOT_COPY", "Copying snapshot %s at revision %d", snapshot.ID, snapshot.Revision)
 
 		for _, chunkHash := range snapshot.FileSequence {
-			chunks[chunkHash] = true  // The chunk is a snapshot chunk
+			chunks[chunkHash] = true // The chunk is a snapshot chunk
 		}
 
 		for _, chunkHash := range snapshot.ChunkSequence {
-			chunks[chunkHash] = true  // The chunk is a snapshot chunk
+			chunks[chunkHash] = true // The chunk is a snapshot chunk
 		}
 
 		for _, chunkHash := range snapshot.LengthSequence {
-			chunks[chunkHash] = true  // The chunk is a snapshot chunk
+			chunks[chunkHash] = true // The chunk is a snapshot chunk
 		}
 
 		description := manager.SnapshotManager.DownloadSequence(snapshot.ChunkSequence)
@@ -1687,7 +1845,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 
 		for _, chunkHash := range snapshot.ChunkHashes {
 			if _, found := chunks[chunkHash]; !found {
-				chunks[chunkHash] = false  // The chunk is a file chunk
+				chunks[chunkHash] = false // The chunk is a file chunk
 			}
 		}
 
@@ -1719,7 +1877,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 		}
 	}
 
-	LOG_INFO("SNAPSHOT_COPY", "Chunks to copy: %d, to skip: %d, total: %d", len(chunksToCopy), len(chunks) - len(chunksToCopy), len(chunks))
+	LOG_INFO("SNAPSHOT_COPY", "Chunks to copy: %d, to skip: %d, total: %d", len(chunksToCopy), len(chunks)-len(chunksToCopy), len(chunks))
 
 	chunkDownloader := CreateChunkOperator(manager.config, manager.storage, nil, false, false, downloadingThreads, false)
 
@@ -1728,7 +1886,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 
 	copiedChunks := 0
 	chunkUploader := CreateChunkOperator(otherManager.config, otherManager.storage, nil, false, false, uploadingThreads, false)
-	chunkUploader.UploadCompletionFunc =  func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
+	chunkUploader.UploadCompletionFunc = func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
 		action := "Skipped"
 		if !skipped {
 			copiedChunks++
@@ -1739,11 +1897,11 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 
 		elapsedTime := time.Now().Sub(startTime).Seconds()
 		speed := int64(float64(atomic.LoadInt64(&uploadedBytes)) / elapsedTime)
-		remainingTime := int64(float64(len(chunksToCopy) - chunkIndex - 1) / float64(chunkIndex + 1) * elapsedTime)
+		remainingTime := int64(float64(len(chunksToCopy)-chunkIndex-1) / float64(chunkIndex+1) * elapsedTime)
-		percentage := float64(chunkIndex + 1) / float64(len(chunksToCopy)) * 100.0
+		percentage := float64(chunkIndex+1) / float64(len(chunksToCopy)) * 100.0
 		LOG_INFO("COPY_PROGRESS", "%s chunk %s (%d/%d) %sB/s %s %.1f%%",
-				action, chunk.GetID(), chunkIndex + 1, len(chunksToCopy),
+			action, chunk.GetID(), chunkIndex+1, len(chunksToCopy),
-				PrettySize(speed), PrettyTime(remainingTime), percentage)
+			PrettySize(speed), PrettyTime(remainingTime), percentage)
 		otherManager.config.PutChunk(chunk)
 	}
 
@@ -1766,7 +1924,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 	chunkDownloader.Stop()
 	chunkUploader.Stop()
 
-	LOG_INFO("SNAPSHOT_COPY", "Copied %d new chunks and skipped %d existing chunks", copiedChunks, len(chunks) - copiedChunks)
+	LOG_INFO("SNAPSHOT_COPY", "Copied %d new chunks and skipped %d existing chunks", copiedChunks, len(chunks)-copiedChunks)
 
 	for _, snapshot := range snapshots {
 		if revisionMap[snapshot.ID][snapshot.Revision] == false {

src/duplicacy_entry.go

@@ -4,8 +4,11 @@
 package duplicacy
 
 import (
+	"bytes"
+	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -15,12 +18,15 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
-	"bytes"
-	"crypto/sha256"
 
-    "github.com/vmihailenco/msgpack"
+	"github.com/vmihailenco/msgpack"
+)
 
+const (
+	entrySymHardLinkRootChunkMarker   = -72
+	entrySymHardLinkTargetChunkMarker = -73
 )
 
 // This is the hidden directory in the repository for storing various files.
@@ -110,15 +116,36 @@ func (entry *Entry) Copy() *Entry {
 		UID: entry.UID,
 		GID: entry.GID,
 
-		StartChunk: entry.StartChunk,
+		StartChunk:  entry.StartChunk,
 		StartOffset: entry.StartOffset,
-		EndChunk: entry.EndChunk,
+		EndChunk:    entry.EndChunk,
-		EndOffset: entry.EndOffset,
+		EndOffset:   entry.EndOffset,
 
 		Attributes: entry.Attributes,
 	}
 }
 
+func (entry *Entry) HardLinkTo(target *Entry, startChunk int, endChunk int) *Entry {
+	return &Entry{
+		Path: entry.Path,
+		Size: target.Size,
+		Time: target.Time,
+		Mode: target.Mode,
+		Link: entry.Link,
+		Hash: target.Hash,
+
+		UID: target.UID,
+		GID: target.GID,
+
+		StartChunk:  startChunk,
+		StartOffset: target.StartOffset,
+		EndChunk:    endChunk,
+		EndOffset:   target.EndOffset,
+
+		Attributes: target.Attributes,
+	}
+}
+
 // CreateEntryFromJSON creates an entry from a json description.
 func (entry *Entry) UnmarshalJSON(description []byte) (err error) {
 
@@ -362,12 +389,12 @@ func (entry *Entry) EncodeMsgpack(encoder *msgpack.Encoder) error {
 
 	if entry.Attributes != nil {
 		attributes := make([]string, numberOfAttributes)
-        i := 0
+		i := 0
-        for attribute := range *entry.Attributes {
+		for attribute := range *entry.Attributes {
-            attributes[i] = attribute
+			attributes[i] = attribute
-            i++
+			i++
-        }
+		}
-        sort.Strings(attributes)
+		sort.Strings(attributes)
 		for _, attribute := range attributes {
 			err = encoder.EncodeString(attribute)
 			if err != nil {
@@ -380,7 +407,7 @@ func (entry *Entry) EncodeMsgpack(encoder *msgpack.Encoder) error {
 		}
 	}
 
-    return nil
+	return nil
 }
 
 func (entry *Entry) DecodeMsgpack(decoder *msgpack.Decoder) error {
@@ -488,18 +515,46 @@ func (entry *Entry) IsLink() bool {
 	return entry.Mode&uint32(os.ModeSymlink) != 0
 }
 
+func (entry *Entry) IsSpecial() bool {
+	return entry.Mode&uint32(os.ModeNamedPipe|os.ModeDevice|os.ModeCharDevice|os.ModeSocket) != 0
+}
+
+func (entry *Entry) IsFileOrSpecial() bool {
+	return entry.Mode&uint32(os.ModeDir|os.ModeSymlink|os.ModeIrregular) == 0
+}
+
 func (entry *Entry) IsComplete() bool {
 	return entry.Size >= 0
 }
 
+func (entry *Entry) IsHardlinkedFrom() bool {
+	return (entry.IsFileOrSpecial() && len(entry.Link) > 0 && entry.Link != "/") || (entry.IsLink() && entry.StartChunk == entrySymHardLinkTargetChunkMarker)
+}
+
+func (entry *Entry) IsHardlinkRoot() bool {
+	return (entry.IsFileOrSpecial() && entry.Link == "/") || (entry.IsLink() && entry.StartChunk == entrySymHardLinkRootChunkMarker)
+}
+
+func (entry *Entry) GetHardlinkId() (int, error) {
+	if entry.IsLink() {
+		if entry.StartChunk != entrySymHardLinkTargetChunkMarker {
+			return 0, errors.New("Symlink entry not marked as hardlinked")
+		}
+		return entry.StartOffset, nil
+	} else {
+		i, err := strconv.ParseUint(entry.Link, 16, 64)
+		return int(i), err
+	}
+}
+
 func (entry *Entry) GetPermissions() os.FileMode {
 	return os.FileMode(entry.Mode) & fileModeMask
 }
 
 func (entry *Entry) GetParent() string {
 	path := entry.Path
-	if path != "" && path[len(path) - 1] == '/' {
+	if path != "" && path[len(path)-1] == '/' {
-		path = path[:len(path) - 1]
+		path = path[:len(path)-1]
 	}
 	i := strings.LastIndex(path, "/")
 	if i == -1 {
@@ -550,6 +605,10 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setO
 		}
 	}
 
+	if entry.Attributes != nil && len(*entry.Attributes) > 0 {
+		entry.SetAttributesToFile(fullPath)
+	}
+
 	// Only set the time if the file is not a symlink
 	if !entry.IsLink() && (*fileInfo).ModTime().Unix() != entry.Time {
 		modifiedTime := time.Unix(entry.Time, 0)
@@ -560,10 +619,6 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setO
 		}
 	}
 
-	if entry.Attributes != nil && len(*entry.Attributes) > 0 {
-		entry.SetAttributesToFile(fullPath)
-	}
-
 	return true
 }
 
@@ -596,7 +651,7 @@ func ComparePaths(left string, right string) int {
 	for i := p; i < len(left); i++ {
 		c3 = left[i]
 		if c3 == '/' {
-			last1 = i == len(left) - 1
+			last1 = i == len(left)-1
 			break
 		}
 	}
@@ -606,7 +661,7 @@ func ComparePaths(left string, right string) int {
 	for i := p; i < len(right); i++ {
 		c4 = right[i]
 		if c4 == '/' {
-			last2 = i == len(right) - 1
+			last2 = i == len(right)-1
 			break
 		}
 	}
@@ -694,10 +749,27 @@ func (files FileInfoCompare) Less(i, j int) bool {
 	}
 }
 
+type listEntryLinkKey struct {
+	dev uint64
+	ino uint64
+}
+
+type ListingState struct {
+	linkIndex int
+	linkTable map[listEntryLinkKey]int // map unique inode details to initially found path
+}
+
+func NewListingState() *ListingState {
+	return &ListingState{
+		linkTable: make(map[listEntryLinkKey]int),
+	}
+}
+
 // ListEntries returns a list of entries representing file and subdirectories under the directory 'path'.  Entry paths
 // are normalized as relative to 'top'.  'patterns' are used to exclude or include certain files.
-func ListEntries(top string, path string, patterns []string, nobackupFile string, excludeByAttribute bool, listingChannel chan *Entry) (directoryList []*Entry,
+func ListEntries(top string, path string, patterns []string, nobackupFile string, excludeByAttribute bool,
-	skippedFiles []string, err error) {
+	listingState *ListingState,
+	listingChannel chan *Entry) (directoryList []*Entry, skippedFiles []string, err error) {
 
 	LOG_DEBUG("LIST_ENTRIES", "Listing %s", path)
 
@@ -735,10 +807,41 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
 		if f.Name() == DUPLICACY_DIRECTORY {
 			continue
 		}
+
 		entry := CreateEntryFromFileInfo(f, normalizedPath)
 		if len(patterns) > 0 && !MatchPath(entry.Path, patterns) {
 			continue
 		}
+
+		var linkKey *listEntryLinkKey
+
+		if runtime.GOOS != "windows" && !entry.IsDir() {
+			if stat := f.Sys().(*syscall.Stat_t); stat != nil && stat.Nlink > 1 {
+				k := listEntryLinkKey{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}
+				if linkIndex, seen := listingState.linkTable[k]; seen {
+					if linkIndex == -1 {
+						LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute (hardlink)", entry.Path)
+						continue
+					}
+					entry.Size = 0
+					if entry.IsLink() {
+						entry.StartChunk = entrySymHardLinkTargetChunkMarker
+						entry.StartOffset = linkIndex
+					} else {
+						entry.Link = strconv.FormatInt(int64(linkIndex), 16)
+					}
+				} else {
+					if entry.IsLink() {
+						entry.StartChunk = entrySymHardLinkRootChunkMarker
+					} else {
+						entry.Link = "/"
+					}
+					listingState.linkTable[k] = -1
+					linkKey = &k
+				}
+			}
+		}
+
 		if entry.IsLink() {
 			isRegular := false
 			isRegular, entry.Link, err = Readlink(joinPath(top, entry.Path))
@@ -769,6 +872,12 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
 				}
 				entry = newEntry
 			}
+		} else if entry.IsSpecial() {
+			if !entry.ReadSpecial(f) {
+				LOG_WARN("LIST_DEV", "Failed to save device node %s", entry.Path)
+				skippedFiles = append(skippedFiles, entry.Path)
+				continue
+			}
 		}
 
 		entry.ReadAttributes(top)
@@ -778,10 +887,9 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
 			continue
 		}
 
-		if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
+		if linkKey != nil {
-			LOG_WARN("LIST_SKIP", "Skipped non-regular file %s", entry.Path)
+			listingState.linkTable[*linkKey] = listingState.linkIndex
-			skippedFiles = append(skippedFiles, entry.Path)
+			listingState.linkIndex++
-			continue
 		}
 
 		if entry.IsDir() {

src/duplicacy_entry_test.go

@@ -5,6 +5,8 @@
 package duplicacy
 
 import (
+	"bytes"
+	"encoding/json"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -13,11 +15,10 @@ import (
 	"sort"
 	"strings"
 	"testing"
-	"bytes"
-	"encoding/json"
 
-	"github.com/gilbertchen/xattr"
+	"github.com/pkg/xattr"
-    "github.com/vmihailenco/msgpack"
+
+	"github.com/vmihailenco/msgpack"
 )
 
 func TestEntrySort(t *testing.T) {
@@ -175,7 +176,7 @@ func TestEntryOrder(t *testing.T) {
 	directories = append(directories, CreateEntry("", 0, 0, 0))
 
 	entries := make([]*Entry, 0, 4)
-    entryChannel := make(chan *Entry, 1024)
+	entryChannel := make(chan *Entry, 1024)
 	entries = append(entries, CreateEntry("", 0, 0, 0))
 
 	for len(directories) > 0 {
@@ -233,8 +234,16 @@ func TestEntryOrder(t *testing.T) {
 // TestEntryExcludeByAttribute tests the excludeByAttribute parameter to the ListEntries function
 func TestEntryExcludeByAttribute(t *testing.T) {
 
-	if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") {
+	var excludeAttrName string
-		t.Skip("skipping test not darwin or linux")
+	var excludeAttrValue []byte
+
+	if runtime.GOOS == "darwin" {
+		excludeAttrName = "com.apple.metadata:com_apple_backup_excludeItem"
+		excludeAttrValue = []byte("com.apple.backupd")
+	} else if runtime.GOOS == "linux" || runtime.GOOS == "freebsd" || runtime.GOOS == "netbsd" || runtime.GOOS == "solaris" {
+		excludeAttrName = "user.duplicacy_exclude"
+	} else {
+		t.Skip("skipping test, not darwin, linux, freebsd, netbsd, or solaris")
 	}
 
 	testDir := filepath.Join(os.TempDir(), "duplicacy_test")
@@ -273,7 +282,7 @@ func TestEntryExcludeByAttribute(t *testing.T) {
 	for _, file := range DATA {
 		fullPath := filepath.Join(testDir, file)
 		if strings.Contains(file, "exclude") {
-			xattr.Setxattr(fullPath, "com.apple.metadata:com_apple_backup_excludeItem", []byte("com.apple.backupd"))
+			xattr.Set(fullPath, excludeAttrName, excludeAttrValue)
 		}
 	}
 
@@ -372,4 +381,4 @@ func TestEntryEncoding(t *testing.T) {
 		t.Error("Decoded entry is different than the original one")
 	}
 
-}
+}

src/duplicacy_entrylist.go

@@ -111,12 +111,12 @@ func (entryList *EntryList)createOnDiskFile() error {
 // Add an entry to the entry list
 func (entryList *EntryList)AddEntry(entry *Entry) error {
 
-	if !entry.IsDir() && !entry.IsLink() {
+	if entry.IsFile() {
 		entryList.NumberOfEntries++
 	}
 
 	if !entry.IsComplete() {
-		if entry.IsDir() || entry.IsLink() {
+		if !entry.IsFile() {
 			entry.Size = 0
 		} else {
 			modifiedEntry := ModifiedEntry {

src/duplicacy_s3storage.go

@@ -90,48 +90,40 @@ func (storage *S3Storage) ListFiles(threadIndex int, dir string) (files []string
 
 	if dir == "snapshots/" {
 		dir = storage.storageDir + dir
-		input := s3.ListObjectsInput{
+		input := s3.ListObjectsV2Input{
 			Bucket:    aws.String(storage.bucket),
 			Prefix:    aws.String(dir),
 			Delimiter: aws.String("/"),
-			MaxKeys:   aws.Int64(1000),
 		}
 
-		output, err := storage.client.ListObjects(&input)
+		err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+			for _, subDir := range page.CommonPrefixes {
+				files = append(files, (*subDir.Prefix)[len(dir):])
+			}
+			return true
+		})
 		if err != nil {
 			return nil, nil, err
 		}
 
-		for _, subDir := range output.CommonPrefixes {
-			files = append(files, (*subDir.Prefix)[len(dir):])
-		}
 		return files, nil, nil
 	} else {
 		dir = storage.storageDir + dir
-		marker := ""
+		input := s3.ListObjectsV2Input{
-		for {
+			Bucket:  aws.String(storage.bucket),
-			input := s3.ListObjectsInput{
+			Prefix:  aws.String(dir),
-				Bucket:  aws.String(storage.bucket),
+			MaxKeys: aws.Int64(1000),
-				Prefix:  aws.String(dir),
+		}
-				MaxKeys: aws.Int64(1000),
-				Marker:  aws.String(marker),
-			}
 
-			output, err := storage.client.ListObjects(&input)
+		err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
-			if err != nil {
+			for _, object := range page.Contents {
-				return nil, nil, err
-			}
-
-			for _, object := range output.Contents {
 				files = append(files, (*object.Key)[len(dir):])
 				sizes = append(sizes, *object.Size)
 			}
-
+			return true
-			if !*output.IsTruncated {
+		})
-				break
+		if err != nil {
-			}
+			return nil, nil, err
-
-			marker = *output.Contents[len(output.Contents)-1].Key
 		}
 		return files, sizes, nil
 	}

src/duplicacy_snapshot.go

@@ -68,6 +68,7 @@ func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
 								         skippedDirectories *[]string, skippedFiles *[]string) {
 
 	var patterns []string
+	listingState := NewListingState()
 
 	if filtersFile == "" {
 		filtersFile = joinPath(GetDuplicacyPreferencePath(), "filters")
@@ -81,7 +82,7 @@ func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
 
 		directory := directories[len(directories)-1]
 		directories = directories[:len(directories)-1]
-		subdirectories, skipped, err := ListEntries(top, directory.Path, patterns, nobackupFile, excludeByAttribute, listingChannel)
+		subdirectories, skipped, err := ListEntries(top, directory.Path, patterns, nobackupFile, excludeByAttribute, listingState, listingChannel)
 		if err != nil {
 			if directory.Path == "" {
 				LOG_ERROR("LIST_FAILURE", "Failed to list the repository root: %v", err)

src/duplicacy_utils.go

@@ -7,14 +7,15 @@ package duplicacy
 import (
 	"bufio"
 	"crypto/sha256"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
-	"runtime"
 
 	"github.com/gilbertchen/gopass"
 	"golang.org/x/crypto/pbkdf2"
@@ -56,7 +57,7 @@ func IsEmptyFilter(pattern string) bool {
 }
 
 func IsUnspecifiedFilter(pattern string) bool {
-	if pattern[0] != '+' && pattern[0] != '-' && !strings.HasPrefix(pattern, "i:") && !strings.HasPrefix(pattern, "e:")  {
+	if pattern[0] != '+' && pattern[0] != '-' && !strings.HasPrefix(pattern, "i:") && !strings.HasPrefix(pattern, "e:") {
 		return true
 	} else {
 		return false
@@ -275,7 +276,6 @@ func SavePassword(preference Preference, passwordType string, password string) {
 // The following code was modified from the online article  'Matching Wildcards: An Algorithm', by Kirk J. Krauss,
 // Dr. Dobb's, August 26, 2008. However, the version in the article doesn't handle cases like matching 'abcccd'
 // against '*ccd', and the version here fixed that issue.
-//
 func matchPattern(text string, pattern string) bool {
 
 	textLength := len(text)
@@ -469,8 +469,39 @@ func PrintMemoryUsage() {
 		runtime.ReadMemStats(&m)
 
 		LOG_INFO("MEMORY_STATS", "Currently allocated: %s, total allocated: %s, system memory: %s, number of GCs: %d",
-		         PrettySize(int64(m.Alloc)), PrettySize(int64(m.TotalAlloc)), PrettySize(int64(m.Sys)), m.NumGC)
+			PrettySize(int64(m.Alloc)), PrettySize(int64(m.TotalAlloc)), PrettySize(int64(m.Sys)), m.NumGC)
 
 		time.Sleep(time.Second)
 	}
-}
+}
+
+func (entry *Entry) dump() map[string]interface{} {
+
+	object := make(map[string]interface{})
+
+	object["path"] = entry.Path
+	object["size"] = entry.Size
+	object["time"] = entry.Time
+	object["mode"] = entry.Mode
+	object["hash"] = entry.Hash
+	object["link"] = entry.Link
+
+	object["content"] = fmt.Sprintf("%d:%d:%d:%d",
+		entry.StartChunk, entry.StartOffset, entry.EndChunk, entry.EndOffset)
+
+	if entry.UID != -1 && entry.GID != -1 {
+		object["uid"] = entry.UID
+		object["gid"] = entry.GID
+	}
+
+	if entry.Attributes != nil && len(*entry.Attributes) > 0 {
+		object["attributes"] = entry.Attributes
+	}
+
+	return object
+}
+
+func (entry *Entry) dumpString() string {
+	data, _ := json.Marshal(entry.dump())
+	return string(data)
+}

src/duplicacy_utils_bsd_common.go

@@ -0,0 +1,109 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+//go:build freebsd || netbsd || darwin
+// +build freebsd netbsd darwin
+
+package duplicacy
+
+import (
+	"bytes"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/pkg/xattr"
+)
+
+const bsdFileFlagsKey = "\x00bf"
+
+func (entry *Entry) ReadAttributes(top string) {
+	fullPath := filepath.Join(top, entry.Path)
+	fileInfo, err := os.Lstat(fullPath)
+	if err != nil {
+		return
+	}
+
+	if !entry.IsSpecial() {
+		attributes, _ := xattr.LList(fullPath)
+		if len(attributes) > 0 {
+			entry.Attributes = &map[string][]byte{}
+			for _, name := range attributes {
+				attribute, err := xattr.LGet(fullPath, name)
+				if err == nil {
+					(*entry.Attributes)[name] = attribute
+				}
+			}
+		}
+	}
+	if err := entry.readFileFlags(fileInfo); err != nil {
+		LOG_INFO("ATTR_BACKUP", "Could not backup flags for file %s: %v", fullPath, err)
+	}
+}
+
+func (entry *Entry) SetAttributesToFile(fullPath string) {
+	if !entry.IsSpecial() {
+		names, _ := xattr.LList(fullPath)
+		for _, name := range names {
+			newAttribute, found := (*entry.Attributes)[name]
+			if found {
+				oldAttribute, _ := xattr.LGet(fullPath, name)
+				if !bytes.Equal(oldAttribute, newAttribute) {
+					xattr.LSet(fullPath, name, newAttribute)
+				}
+				delete(*entry.Attributes, name)
+			} else {
+				xattr.LRemove(fullPath, name)
+			}
+		}
+
+		for name, attribute := range *entry.Attributes {
+			if len(name) > 0 && name[0] == '\x00' {
+				continue
+			}
+			xattr.LSet(fullPath, name, attribute)
+		}
+	}
+	if err := entry.restoreLateFileFlags(fullPath); err != nil {
+		LOG_DEBUG("ATTR_RESTORE", "Could not restore flags for file %s: %v", fullPath, err)
+	}
+}
+
+func (entry *Entry) readFileFlags(fileInfo os.FileInfo) error {
+	stat, ok := fileInfo.Sys().(*syscall.Stat_t)
+	if ok && stat.Flags != 0 {
+		if entry.Attributes == nil {
+			entry.Attributes = &map[string][]byte{}
+		}
+		v := make([]byte, 4)
+		binary.LittleEndian.PutUint32(v, stat.Flags)
+		(*entry.Attributes)[bsdFileFlagsKey] = v
+		LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", stat.Flags, entry.Path)
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyDirFlags(path string) error {
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
+	return nil
+}
+
+func (entry *Entry) RestoreSpecial(fullPath string) error {
+	mode := entry.Mode & uint32(fileModeMask)
+
+	if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
+		mode |= syscall.S_IFIFO
+	} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
+		mode |= syscall.S_IFCHR
+	} else if entry.Mode&uint32(os.ModeDevice) != 0 {
+		mode |= syscall.S_IFBLK
+	} else {
+		return nil
+	}
+	return syscall.Mknod(fullPath, mode, int(entry.GetRdev()))
+}

src/duplicacy_utils_darwin.go

@@ -5,10 +5,29 @@
 package duplicacy
 
 import (
+	"encoding/binary"
+	"os"
 	"strings"
+	"syscall"
 )
 
-func excludedByAttribute(attirbutes map[string][]byte) bool {
+func excludedByAttribute(attributes map[string][]byte) bool {
-	value, ok := attirbutes["com.apple.metadata:com_apple_backup_excludeItem"]
+	value, ok := attributes["com.apple.metadata:com_apple_backup_excludeItem"]
 	return ok && strings.Contains(string(value), "com.apple.backupd")
 }
+
+func (entry *Entry) restoreLateFileFlags(path string) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
+		f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_SYMLINK, 0)
+		if err != nil {
+			return err
+		}
+		err = syscall.Fchflags(int(f.Fd()), int(binary.LittleEndian.Uint32(v)))
+		f.Close()
+		return err
+	}
+	return nil
+}

src/duplicacy_utils_linux.go

@@ -5,9 +5,238 @@
 package duplicacy
 
 import (
+	"bytes"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+	"syscall"
+	"unsafe"
+
+	"github.com/pkg/xattr"
 )
 
-func excludedByAttribute(attirbutes map[string][]byte) bool {
+const (
-	_, ok := attirbutes["duplicacy_exclude"]
+	linux_FS_SECRM_FL        = 0x00000001 /* Secure deletion */
+	linux_FS_UNRM_FL         = 0x00000002 /* Undelete */
+	linux_FS_COMPR_FL        = 0x00000004 /* Compress file */
+	linux_FS_SYNC_FL         = 0x00000008 /* Synchronous updates */
+	linux_FS_IMMUTABLE_FL    = 0x00000010 /* Immutable file */
+	linux_FS_APPEND_FL       = 0x00000020 /* writes to file may only append */
+	linux_FS_NODUMP_FL       = 0x00000040 /* do not dump file */
+	linux_FS_NOATIME_FL      = 0x00000080 /* do not update atime */
+	linux_FS_NOCOMP_FL       = 0x00000400 /* Don't compress */
+	linux_FS_JOURNAL_DATA_FL = 0x00004000 /* Reserved for ext3 */
+	linux_FS_NOTAIL_FL       = 0x00008000 /* file tail should not be merged */
+	linux_FS_DIRSYNC_FL      = 0x00010000 /* dirsync behaviour (directories only) */
+	linux_FS_TOPDIR_FL       = 0x00020000 /* Top of directory hierarchies*/
+	linux_FS_NOCOW_FL        = 0x00800000 /* Do not cow file */
+	linux_FS_PROJINHERIT_FL  = 0x20000000 /* Create with parents projid */
+
+	linux_FS_IOC_GETFLAGS uintptr = 0x80086601
+	linux_FS_IOC_SETFLAGS uintptr = 0x40086602
+
+	linuxIocFlagsFileEarly = linux_FS_SECRM_FL | linux_FS_UNRM_FL | linux_FS_COMPR_FL | linux_FS_NODUMP_FL | linux_FS_NOATIME_FL | linux_FS_NOCOMP_FL | linux_FS_JOURNAL_DATA_FL | linux_FS_NOTAIL_FL | linux_FS_NOCOW_FL
+	linuxIocFlagsDirEarly  = linux_FS_TOPDIR_FL | linux_FS_PROJINHERIT_FL
+	linuxIocFlagsLate      = linux_FS_SYNC_FL | linux_FS_IMMUTABLE_FL | linux_FS_APPEND_FL | linux_FS_DIRSYNC_FL
+
+	linuxFileFlagsKey = "\x00lf"
+)
+
+func ioctl(f *os.File, request uintptr, attrp *uint32) error {
+	argp := uintptr(unsafe.Pointer(attrp))
+
+	if _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), request, argp); errno != 0 {
+		return os.NewSyscallError("ioctl", errno)
+	}
+	return nil
+}
+
+type xattrHandle struct {
+	f        *os.File
+	fullPath string
+}
+
+func (x xattrHandle) list() ([]string, error) {
+	if x.f != nil {
+		return xattr.FList(x.f)
+	} else {
+		return xattr.LList(x.fullPath)
+	}
+}
+
+func (x xattrHandle) get(name string) ([]byte, error) {
+	if x.f != nil {
+		return xattr.FGet(x.f, name)
+	} else {
+		return xattr.LGet(x.fullPath, name)
+	}
+}
+
+func (x xattrHandle) set(name string, value []byte) error {
+	if x.f != nil {
+		return xattr.FSet(x.f, name, value)
+	} else {
+		return xattr.LSet(x.fullPath, name, value)
+	}
+}
+
+func (x xattrHandle) remove(name string) error {
+	if x.f != nil {
+		return xattr.FRemove(x.f, name)
+	} else {
+		return xattr.LRemove(x.fullPath, name)
+	}
+}
+
+func (entry *Entry) ReadAttributes(top string) {
+	fullPath := filepath.Join(top, entry.Path)
+	x := xattrHandle{nil, fullPath}
+
+	if !entry.IsLink() {
+		var err error
+		x.f, err = os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
+		if err != nil {
+			// FIXME: We really should return errors for failure to read
+			return
+		}
+	}
+
+	attributes, _ := x.list()
+
+	if len(attributes) > 0 {
+		entry.Attributes = &map[string][]byte{}
+	}
+	for _, name := range attributes {
+		attribute, err := x.get(name)
+		if err == nil {
+			(*entry.Attributes)[name] = attribute
+		}
+	}
+
+	if entry.IsFile() || entry.IsDir() {
+		if err := entry.readFileFlags(x.f); err != nil {
+			LOG_INFO("ATTR_BACKUP", "Could not backup flags for file %s: %v", fullPath, err)
+		}
+	}
+	x.f.Close()
+}
+
+func (entry *Entry) SetAttributesToFile(fullPath string) {
+	x := xattrHandle{nil, fullPath}
+	if !entry.IsLink() {
+		var err error
+		x.f, err = os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
+		if err != nil {
+			return
+		}
+	}
+
+	names, _ := x.list()
+
+	for _, name := range names {
+		newAttribute, found := (*entry.Attributes)[name]
+		if found {
+			oldAttribute, _ := x.get(name)
+			if !bytes.Equal(oldAttribute, newAttribute) {
+				x.set(name, newAttribute)
+			}
+			delete(*entry.Attributes, name)
+		} else {
+			x.remove(name)
+		}
+	}
+
+	for name, attribute := range *entry.Attributes {
+		if len(name) > 0 && name[0] == '\x00' {
+			continue
+		}
+		x.set(name, attribute)
+	}
+	if entry.IsFile() || entry.IsDir() {
+		if err := entry.restoreLateFileFlags(x.f); err != nil {
+			LOG_DEBUG("ATTR_RESTORE", "Could not restore flags for file %s: %v", fullPath, err)
+		}
+	}
+	x.f.Close()
+}
+
+func (entry *Entry) readFileFlags(f *os.File) error {
+	var flags uint32
+	if err := ioctl(f, linux_FS_IOC_GETFLAGS, &flags); err != nil {
+		return err
+	}
+	if flags != 0 {
+		if entry.Attributes == nil {
+			entry.Attributes = &map[string][]byte{}
+		}
+		v := make([]byte, 4)
+		binary.LittleEndian.PutUint32(v, flags)
+		(*entry.Attributes)[linuxFileFlagsKey] = v
+		LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", flags, entry.Path)
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyDirFlags(path string) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
+		flags := binary.LittleEndian.Uint32(v) & linuxIocFlagsDirEarly
+		f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_DIRECTORY, 0)
+		if err != nil {
+			return err
+		}
+		LOG_DEBUG("ATTR_RESTORE", "Restore dir flags (early) 0x%x for %s", flags, entry.Path)
+		err = ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
+		f.Close()
+		return err
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
+		flags := binary.LittleEndian.Uint32(v) & linuxIocFlagsFileEarly
+		LOG_DEBUG("ATTR_RESTORE", "Restore flags (early) 0x%x for %s", flags, entry.Path)
+		return ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
+	}
+	return nil
+}
+
+func (entry *Entry) restoreLateFileFlags(f *os.File) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
+		flags := binary.LittleEndian.Uint32(v) & (linuxIocFlagsFileEarly | linuxIocFlagsDirEarly | linuxIocFlagsLate)
+		LOG_DEBUG("ATTR_RESTORE", "Restore flags (late) 0x%x for %s", flags, entry.Path)
+		return ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreSpecial(fullPath string) error {
+	mode := entry.Mode & uint32(fileModeMask)
+
+	if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
+		mode |= syscall.S_IFIFO
+	} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
+		mode |= syscall.S_IFCHR
+	} else if entry.Mode&uint32(os.ModeDevice) != 0 {
+		mode |= syscall.S_IFBLK
+	} else if entry.Mode&uint32(os.ModeSocket) != 0 {
+		mode |= syscall.S_IFSOCK
+	} else {
+		return nil
+	}
+	return syscall.Mknod(fullPath, mode, int(entry.GetRdev()))
+}
+
+func excludedByAttribute(attributes map[string][]byte) bool {
+	_, ok := attributes["user.duplicacy_exclude"]
 	return ok
 }

src/duplicacy_utils_others.go

@@ -2,18 +2,17 @@
 // Free for personal use and commercial trial
 // Commercial use requires per-user licenses available from https://duplicacy.com
 
+//go:build !windows
 // +build !windows
 
 package duplicacy
 
 import (
-	"bytes"
 	"os"
 	"path"
-	"path/filepath"
 	"syscall"
 
-	"github.com/pkg/xattr"
+	"golang.org/x/sys/unix"
 )
 
 func Readlink(path string) (isRegular bool, s string, err error) {
@@ -47,43 +46,35 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
 	return true
 }
 
-func (entry *Entry) ReadAttributes(top string) {
+func (entry *Entry) ReadSpecial(fileInfo os.FileInfo) bool {
-
+	if fileInfo.Mode()&(os.ModeDevice|os.ModeCharDevice) == 0 {
-	fullPath := filepath.Join(top, entry.Path)
+		return true
-	attributes, _ := xattr.List(fullPath)
-	if len(attributes) > 0 {
-		entry.Attributes = &map[string][]byte{}
-		for _, name := range attributes {
-			attribute, err := xattr.Get(fullPath, name)
-			if err == nil {
-				(*entry.Attributes)[name] = attribute
-			}
-		}
 	}
+	stat := fileInfo.Sys().(*syscall.Stat_t)
+	if stat == nil {
+		return false
+	}
+	entry.Size = 0
+	rdev := uint64(stat.Rdev)
+	entry.StartChunk = int(rdev & 0xFFFFFFFF)
+	entry.StartOffset = int(rdev >> 32)
+	return true
 }
 
-func (entry *Entry) SetAttributesToFile(fullPath string) {
+func (entry *Entry) GetRdev() uint64 {
-	names, _ := xattr.List(fullPath)
+	return uint64(entry.StartChunk) | uint64(entry.StartOffset)<<32
+}
 
-	for _, name := range names {
+func (entry *Entry) IsSameSpecial(fileInfo os.FileInfo) bool {
-
+	stat := fileInfo.Sys().(*syscall.Stat_t)
-
+	if stat == nil {
-		newAttribute, found := (*entry.Attributes)[name]
+		return false
-		if found {
-			oldAttribute, _ := xattr.Get(fullPath, name)
-			if !bytes.Equal(oldAttribute, newAttribute) {
-				xattr.Set(fullPath, name, newAttribute)
-			}
-			delete(*entry.Attributes, name)
-		} else {
-			xattr.Remove(fullPath, name)
-		}
-	}
-
-	for name, attribute := range *entry.Attributes {
-		xattr.Set(fullPath, name, attribute)
 	}
+	return (uint32(fileInfo.Mode()) == entry.Mode) && (uint64(stat.Rdev) == entry.GetRdev())
+}
 
+func MakeHardlink(source string, target string) error {
+	return unix.Linkat(unix.AT_FDCWD, source, unix.AT_FDCWD, target, 0)
 }
 
 func joinPath(components ...string) string {

src/duplicacy_utils_posix.go

@@ -2,12 +2,12 @@
 // Free for personal use and commercial trial
 // Commercial use requires per-user licenses available from https://duplicacy.com
 
+//go:build freebsd || netbsd || solaris
+// +build freebsd netbsd solaris
+
 package duplicacy
 
-import (
+func excludedByAttribute(attributes map[string][]byte) bool {
-)
+	_, ok := attributes["user.duplicacy_exclude"]
-
-func excludedByAttribute(attirbutes map[string][]byte) bool {
-	_, ok := attirbutes["duplicacy_exclude"]
 	return ok
-}
+}

src/duplicacy_utils_windows.go

@@ -117,6 +117,18 @@ func (entry *Entry) SetAttributesToFile(fullPath string) {
 
 }
 
+func (entry *Entry) ReadDeviceNode(fileInfo os.FileInfo) bool {
+	return nil
+}
+
+func (entry *Entry) RestoreSpecial(fullPath string) error {
+	return nil
+}
+
+func MakeHardlink(source string, target string) error {
+	return os.Link(source, target)
+}
+
 func joinPath(components ...string) string {
 
 	combinedPath := `\\?\` + filepath.Join(components...)
@@ -132,6 +144,18 @@ func SplitDir(fullPath string) (dir string, file string) {
 	return fullPath[:i+1], fullPath[i+1:]
 }
 
-func excludedByAttribute(attirbutes map[string][]byte) bool {
+func (entry *Entry) ReadFileFlags(f *os.File) error {
-	return false
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyDirFlags(path string) error {
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
+	return nil
+}
+
+func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
+	return nil
 }

src/duplicacy_utils_xbsd.go

@@ -0,0 +1,30 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+//go:build freebsd || netbsd
+// +build freebsd netbsd
+
+package duplicacy
+
+import (
+	"bytes"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/pkg/xattr"
+)
+
+func (entry *Entry) restoreLateFileFlags(path string) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
+		if _, _, errno := syscall.Syscall(syscall.SYS_LCHFLAGS, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(v), 0); errno != 0 {
+			return os.NewSyscallError("lchflags", errno)
+		}
+	}
+	return nil
+}