Support backup of sockets

This only works on Linux. Darwin does not allow mknod of a socket. Have not tested BSD.
gofmt some files
2026-01-02 11:44:45 -06:00 · 2023-10-04 02:53:48 -05:00 · 2023-10-04 02:53:48 -05:00 · 2023-10-04 02:53:48 -05:00 · 2023-10-04 02:53:48 -05:00 · 2023-10-04 02:53:48 -05:00
14 changed files with 776 additions and 213 deletions
--- a/duplicacy/duplicacy_main.go
+++ b/duplicacy/duplicacy_main.go
@@ -2262,7 +2262,7 @@ func main() {
 	app.Name = "duplicacy"
 	app.HelpName = "duplicacy"
 	app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
-	app.Version = "3.2.1" + " (" + GitCommit + ")"
+	app.Version = "3.2.2" + " (" + GitCommit + ")"

 	// Exit with code 2 if an invalid command is provided
 	app.CommandNotFound = func(context *cli.Context, command string) {
--- a/src/duplicacy_backupmanager.go
+++ b/src/duplicacy_backupmanager.go
@@ -21,7 +21,7 @@ import (
 	"sync/atomic"
 	"time"

-    "github.com/vmihailenco/msgpack"
+	"github.com/vmihailenco/msgpack"
 )

 // BackupManager performs the two major operations, backup and restore, and passes other operations, mostly related to
@@ -36,9 +36,9 @@ type BackupManager struct {

 	config *Config // contains a number of options

-	nobackupFile string // don't backup directory when this file name is found
-	filtersFile string  // the path to the filters file
-	excludeByAttribute bool // don't backup file based on file attribute
+	nobackupFile       string // don't backup directory when this file name is found
+	filtersFile        string // the path to the filters file
+	excludeByAttribute bool   // don't backup file based on file attribute

 	cachePath string
 }
@@ -146,7 +146,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta

 	if manager.config.DataShards != 0 && manager.config.ParityShards != 0 {
 		LOG_INFO("BACKUP_ERASURECODING", "Erasure coding is enabled with %d data shards and %d parity shards",
-		         manager.config.DataShards, manager.config.ParityShards)
+			manager.config.DataShards, manager.config.ParityShards)
 	}

 	if manager.config.rsaPublicKey != nil && len(manager.config.FileKey) > 0 {
@@ -187,7 +187,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta

 	// If the listing operation is fast and this is an initial backup, list all chunks and
 	// put them in the cache.
-	if (manager.storage.IsFastListing() && remoteSnapshot.Revision == 0) {
+	if manager.storage.IsFastListing() && remoteSnapshot.Revision == 0 {
 		LOG_INFO("BACKUP_LIST", "Listing all chunks")
 		allChunks, _ := manager.SnapshotManager.ListAllFiles(manager.storage, "chunks/")

@@ -222,7 +222,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta

 	var totalModifiedFileSize int64    // total size of modified files
 	var uploadedModifiedFileSize int64 // portions that have been uploaded (including cache hits)
-	var preservedFileSize int64 // total size of unmodified files
+	var preservedFileSize int64        // total size of unmodified files

 	localSnapshot := CreateEmptySnapshot(manager.snapshotID)
 	localSnapshot.Revision = remoteSnapshot.Revision + 1
@@ -239,7 +239,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 		// List local files
 		defer CatchLogException()
 		localSnapshot.ListLocalFiles(shadowTop, manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute, localListingChannel, &skippedDirectories, &skippedFiles)
-	} ()
+	}()

 	go func() {
 		// List remote files
@@ -261,7 +261,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 			})
 		}
 		close(remoteListingChannel)
-	} ()
+	}()

 	// Create the local file list
 	localEntryList, err := CreateEntryList(manager.snapshotID, manager.cachePath, maximumInMemoryEntries)
@@ -275,7 +275,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 	var remoteEntry *Entry
 	remoteListingOK := true
 	for {
-		localEntry := <- localListingChannel
+		localEntry := <-localListingChannel
 		if localEntry == nil {
 			break
 		}
@@ -289,7 +289,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 				compareResult = localEntry.Compare(remoteEntry)
 			} else {
 				if remoteListingOK {
-					remoteEntry, remoteListingOK = <- remoteListingChannel
+					remoteEntry, remoteListingOK = <-remoteListingChannel
 				}
 				if !remoteListingOK {
 					compareResult = -1
@@ -304,24 +304,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 			remoteEntry = nil
 		}

-		if localEntry.IsHardlinkedFrom() {
-			// FIXME: Sanity check?
-			// FIXME: perhaps we can make size = 0 an initial invariant of the link?
-			//
-			// Note that if the initial size was 0 then this original logic doesn't change!
-			localEntry.Size = 0
-			// targetEntry, ok := localEntryList.HardLinkTable[localEntry.Link]
-			// if !ok {
-			// 	LOG_ERROR("BACKUP_CREATE", "Hard link %s not found in entry cache for path %s", localEntry.Link, localEntry.Path)
-			// }
-			// localEntry.Size = targetEntry.Size
-			// localEntry.Hash = targetEntry.Hash
-			// localEntry.StartChunk = targetEntry.StartChunk
-			// localEntry.StartOffset = targetEntry.StartOffset
-			// localEntry.EndChunk = targetEntry.EndChunk
-			// localEntry.EndOffset = targetEntry.EndOffset
-			// LOG_DEBUG("BACKUP_CREATE", "Hard link %s to %s in initial listing", localEntry.Link, targetEntry.Path)
-		} else if compareResult == 0  {
+		if compareResult == 0 {
 			// No need to check if it is in hash mode -- in that case remote listing is nil
 			if localEntry.IsSameAs(remoteEntry) && localEntry.IsFile() {
 				if localEntry.Size > 0 {
@@ -356,8 +339,8 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 			// compareResult must be < 0; the local file is new
 			totalModifiedFileSize += localEntry.Size
 			if localEntry.Size > 0 {
-					// A size of -1 indicates this is a modified file that will be uploaded
-					localEntry.Size = -1
+				// A size of -1 indicates this is a modified file that will be uploaded
+				localEntry.Size = -1
 			}
 		}

@@ -465,7 +448,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta

 		_, found := chunkCache[chunkID]
 		if found {
-			if time.Now().Unix() - lastUploadingTime > keepUploadAlive {
+			if time.Now().Unix()-lastUploadingTime > keepUploadAlive {
 				LOG_INFO("UPLOAD_KEEPALIVE", "Skip chunk cache to keep connection alive")
 				found = false
 			}
@@ -575,7 +558,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 	if showStatistics {

 		LOG_INFO("BACKUP_STATS", "Files: %d total, %s bytes; %d new, %s bytes",
-			localEntryList.NumberOfEntries - int64(len(skippedFiles)),
+			localEntryList.NumberOfEntries-int64(len(skippedFiles)),
 			PrettyNumber(preservedFileSize+uploadedFileSize),
 			len(localEntryList.ModifiedEntries), PrettyNumber(uploadedFileSize))

@@ -639,11 +622,6 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 	return true
 }

-type hardLinkEntry struct {
-	entry *Entry
-	willDownload bool
-}
-
 // Restore downloads the specified snapshot, compares it with what's on the repository, and then downloads
 // files that are different. 'base' is a directory that contains files at a different revision which can
 // serve as a local cache to avoid download chunks available locally.  It is perfectly ok for 'base' to be
@@ -708,7 +686,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		// List local files
 		defer CatchLogException()
 		localSnapshot.ListLocalFiles(top, manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute, localListingChannel, nil, nil)
-	} ()
+	}()

 	remoteSnapshot := manager.SnapshotManager.DownloadSnapshot(manager.snapshotID, revision)
 	manager.SnapshotManager.DownloadSnapshotSequences(remoteSnapshot)
@@ -720,19 +698,45 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 			return true
 		})
 		close(remoteListingChannel)
-	} ()
+	}()

 	var localEntry *Entry
 	localListingOK := true

-	hardLinkTable := make(map[string]hardLinkEntry)
-	//hardLinks := make([]*Entry, 0)
+	type hardLinkEntry struct {
+		entry     *Entry
+		willExist bool
+	}
+	var hardLinkTable []hardLinkEntry
+	var hardLinks []*Entry
+
+	restoreHardlink := func(entry *Entry, fullPath string) bool {
+		if entry.IsHardlinkRoot() {
+			hardLinkTable[len(hardLinkTable)-1].willExist = true
+		} else if entry.IsHardlinkedFrom() {
+			i, err := entry.GetHardlinkId()
+			if err != nil {
+				LOG_ERROR("RESTORE_HARDLINK", "Decode error for hardlinked entry %s, %v", entry.Path, err)
+				return false
+			}
+			if !hardLinkTable[i].willExist {
+				hardLinkTable[i] = hardLinkEntry{entry, true}
+			} else {
+				sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
+				LOG_INFO("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath)
+				if err := MakeHardlink(sourcePath, fullPath); err != nil {
+					LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s %v", fullPath, sourcePath, err)
+				}
+				return true
+			}
+		}
+		return false
+	}

 	for remoteEntry := range remoteListingChannel {

-		if remoteEntry.IsFile() && remoteEntry.Link == "/" {
-			LOG_INFO("RESTORE_LINK", "Noting hardlinked source file %s", remoteEntry.Path)
-			hardLinkTable[remoteEntry.Path] = hardLinkEntry{remoteEntry, false}
+		if remoteEntry.IsHardlinkRoot() {
+			hardLinkTable = append(hardLinkTable, hardLinkEntry{remoteEntry, false})
 		}

 		if len(patterns) > 0 && !MatchPath(remoteEntry.Path, patterns) {
@@ -743,10 +747,8 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		var compareResult int

 		for {
-			// TODO: We likely need to check if a local listing file exists in the hardLinkTable for the case where one is restoring a hardlink
-			// to an existing disk file. Right now, we'll just end up downloading the file new.
 			if localEntry == nil && localListingOK {
-				localEntry, localListingOK = <- localListingChannel
+				localEntry, localListingOK = <-localListingChannel
 			}
 			if localEntry == nil {
 				compareResult = 1
@@ -762,53 +764,50 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		}

 		if compareResult == 0 {
-			// if quickMode && localEntry.IsFile() {
-				if quickMode && localEntry.IsFile() && localEntry.IsSameAs(remoteEntry) {
-					LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", localEntry.Path)
-					skippedFileSize += localEntry.Size
-					skippedFileCount++
-					localEntry = nil
-					continue
-				// checkEntry := remoteEntry
-				// if len(remoteEntry.Link) > 0 && remoteEntry.Link != "/" {
-				// 	if e, ok := hardLinkTable[remoteEntry.Link]; !ok {
-				// 		LOG_ERROR("RESTORE_LINK", "Source file %s for hardlink %s missing", remoteEntry.Link, remoteEntry.Path)
-				// 	} else {
-				// 		checkEntry = e.entry
-				// 	}
-				// }
-				// if localEntry.IsSameAs(checkEntry) {
-				// 	LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", localEntry.Path)
-				// 	skippedFileSize += localEntry.Size
-				// 	skippedFileCount++
-				// 	localEntry = nil
-				// 	continue
-				// }
+			if quickMode && localEntry.IsFile() && localEntry.IsSameAs(remoteEntry) {
+				LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", localEntry.Path)
+				skippedFileSize += localEntry.Size
+				skippedFileCount++
+				localEntry = nil
+				continue
 			}
 			localEntry = nil
 		}

 		fullPath := joinPath(top, remoteEntry.Path)
+
 		if remoteEntry.IsLink() {
-			stat, err := os.Lstat(fullPath)
-			if stat != nil {
+			if stat, _ := os.Lstat(fullPath); stat != nil {
 				if stat.Mode()&os.ModeSymlink != 0 {
 					isRegular, link, err := Readlink(fullPath)
 					if err == nil && link == remoteEntry.Link && !isRegular {
 						remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
+						if remoteEntry.IsHardlinkRoot() {
+							hardLinkTable[len(hardLinkTable)-1].willExist = true
+						}
 						continue
 					}
 				}

+				if !overwrite {
+					LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
+						"File %s already exists.  Please specify the -overwrite option to overwrite", remoteEntry.Path)
+					continue
+				}
+
 				os.Remove(fullPath)
 			}

-			err = os.Symlink(remoteEntry.Link, fullPath)
-			if err != nil {
+			if restoreHardlink(remoteEntry, fullPath) {
+				continue
+			}
+
+			if err := os.Symlink(remoteEntry.Link, fullPath); err != nil {
 				LOG_ERROR("RESTORE_SYMLINK", "Can't create symlink %s: %v", remoteEntry.Path, err)
 				return 0
 			}
 			remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
+
 			LOG_TRACE("DOWNLOAD_DONE", "Symlink %s updated", remoteEntry.Path)
 		} else if remoteEntry.IsDir() {

@@ -828,23 +827,50 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 					return 0
 				}
 			}
+			remoteEntry.RestoreEarlyDirFlags(fullPath)
 			directoryEntries = append(directoryEntries, remoteEntry)
+		} else if remoteEntry.IsSpecial() {
+			if stat, _ := os.Lstat(fullPath); stat != nil {
+				if remoteEntry.IsSameSpecial(stat) {
+					remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
+					if remoteEntry.IsHardlinkRoot() {
+						hardLinkTable[len(hardLinkTable)-1].willExist = true
+					}
+				}
+				if !overwrite {
+					LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
+						"File %s already exists.  Please specify the -overwrite option to overwrite", remoteEntry.Path)
+					continue
+				}
+				os.Remove(fullPath)
+			}
+
+			if restoreHardlink(remoteEntry, fullPath) {
+				continue
+			}
+
+			if err := remoteEntry.RestoreSpecial(fullPath); err != nil {
+				LOG_ERROR("RESTORE_SPECIAL", "Unable to restore special file %s: %v", remoteEntry.Path, err)
+				return 0
+			}
+			remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
 		} else {
-			// if remoteEntry.Link == "/" {
-			// 	hardLinkTable[remoteEntry.Path] = hardLinkEntry{remoteEntry, true}
-			// } else if len(remoteEntry.Link) > 0 {
-			// 	if e, ok := hardLinkTable[remoteEntry.Link]; !ok {
-			// 		LOG_ERROR("RESTORE_LINK", "Source file %s for hardlink %s missing", remoteEntry.Link, remoteEntry.Path)
-			// 	} else if !e.willDownload {
-			// 		origSourcePath := e.entry.Path
-			// 		e.entry.Path = remoteEntry.Path
-			// 		remoteEntry = e.entry
-			// 		hardLinkTable[origSourcePath] = hardLinkEntry{remoteEntry, true}
-			// 	} else {
-			// 		hardLinks = append(hardLinks, remoteEntry)
-			// 		continue
-			// 	}
-			// }
+			if remoteEntry.IsHardlinkRoot() {
+				hardLinkTable[len(hardLinkTable)-1].willExist = true
+			} else if remoteEntry.IsHardlinkedFrom() {
+				i, err := remoteEntry.GetHardlinkId()
+				if err != nil {
+					LOG_ERROR("RESTORE_HARDLINK", "Decode error for hardlinked entry %s, %v", remoteEntry.Path, err)
+					return 0
+				}
+				if !hardLinkTable[i].willExist {
+					hardLinkTable[i] = hardLinkEntry{remoteEntry, true}
+				} else {
+					hardLinks = append(hardLinks, remoteEntry)
+					continue
+				}
+			}
+
 			// We can't download files here since fileEntries needs to be sorted
 			fileEntries = append(fileEntries, remoteEntry)
 			totalFileSize += remoteEntry.Size
@@ -856,7 +882,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 	}

 	for localListingOK {
-		localEntry, localListingOK = <- localListingChannel
+		localEntry, localListingOK = <-localListingChannel
 		if localEntry != nil {
 			extraFiles = append(extraFiles, localEntry.Path)
 		}
@@ -900,10 +926,6 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		stat, _ := os.Stat(fullPath)
 		if stat != nil {
 			if quickMode {
-				// cmpFile := file
-				// if file.IsFile() && len(file.Link) > 0 && file.Link != "/" {
-				// 	cmpFile = hardLinkTable[file.Link].entry
-				// }
 				if file.IsSameAsFileInfo(stat) {
 					LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", file.Path)
 					skippedFileSize += file.Size
@@ -941,6 +963,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 				downloadedFileSize += file.Size
 				downloadedFiles = append(downloadedFiles, file)
 			}
+
 			continue
 		}

@@ -967,14 +990,37 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		file.RestoreMetadata(fullPath, nil, setOwner)
 	}

-	// for _, linkEntry := range hardLinks {
-	// 	sourcePath := joinPath(top, hardLinkTable[linkEntry.Link].entry.Path)
-	// 	fullPath := joinPath(top, linkEntry.Path)
-	// 	LOG_INFO("DOWNLOAD_LINK", "Hard linking %s -> %s", fullPath, sourcePath)
-	// 	if err := os.Link(sourcePath, fullPath); err != nil {
-	// 		LOG_ERROR("DOWNLOAD_LINK", "Failed to create hard link %s -> %s", fullPath, sourcePath)
-	// 	}
-	// }
+	for _, linkEntry := range hardLinks {
+
+		i, _ := linkEntry.GetHardlinkId()
+		sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
+		fullPath := joinPath(top, linkEntry.Path)
+
+		if stat, _ := os.Lstat(fullPath); stat != nil {
+			sourceStat, _ := os.Lstat(sourcePath)
+			if os.SameFile(stat, sourceStat) {
+				continue
+			}
+
+			if sourceStat == nil {
+				LOG_WERROR(allowFailures, "RESTORE_HARDLINK",
+					"Target %s for hardlink %s is missing", sourcePath, linkEntry.Path)
+				continue
+			}
+			if !overwrite {
+				LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
+					"File %s already exists.  Please specify the -overwrite option to overwrite", linkEntry.Path)
+				continue
+			}
+			os.Remove(fullPath)
+		}
+
+		LOG_DEBUG("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath)
+		if err := MakeHardlink(sourcePath, fullPath); err != nil {
+			LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s", fullPath, sourcePath)
+			return 0
+		}
+	}

 	if deleteMode && len(patterns) == 0 {
 		// Reverse the order to make sure directories are empty before being deleted
@@ -1121,17 +1167,19 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
 	encoder := msgpack.NewEncoder(buffer)
 	metadataChunkMaker := CreateMetaDataChunkMaker(manager.config, metadataChunkSize)

-    var chunkHashes  []string
+	var chunkHashes []string
 	var chunkLengths []int
 	lastChunk := -1

 	lastEndChunk := 0

-	uploadEntryInfoFunc := func(entry *Entry) error {
-		if entry.IsHardlinkRoot() {
-			entryList.HardLinkTable[entry.Path] = entry
-		}
+	type hardLinkEntry struct {
+		entry      *Entry
+		startChunk int
+	}
+	var hardLinkTable []hardLinkEntry

+	uploadEntryInfoFunc := func(entry *Entry) error {
 		if entry.IsFile() && entry.Size > 0 {
 			delta := entry.StartChunk - len(chunkHashes) + 1
 			if entry.StartChunk != lastChunk {
@@ -1149,18 +1197,38 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
 			entry.StartChunk -= delta
 			entry.EndChunk -= delta

+			if entry.IsHardlinkRoot() {
+				LOG_DEBUG("SNAPSHOT_UPLOAD", "Hard link root %s %v %v", entry.Path, entry.StartChunk, entry.EndChunk)
+				hardLinkTable = append(hardLinkTable, hardLinkEntry{entry, entry.StartChunk})
+			}
+
 			delta = entry.EndChunk - entry.StartChunk
 			entry.StartChunk -= lastEndChunk
 			lastEndChunk = entry.EndChunk
 			entry.EndChunk = delta
-		} else if entry.IsHardlinkedFrom() {
-			targetEntry, ok := entryList.HardLinkTable[entry.Link]
-			if !ok {
-				LOG_ERROR("SNAPSHOT_UPLOAD", "Unable to find hardlink target for %s to %s", entry.Path, entry.Link)
+		} else if entry.IsHardlinkedFrom() && !entry.IsLink() {
+			i, err := entry.GetHardlinkId()
+			if err != nil {
+				LOG_ERROR("SNAPSHOT_UPLOAD", "Decode error for hardlinked entry %s, %v", entry.Link, err)
+				return err
 			}
-			// FIXME: We will use a copy, so it is probably sufficient to skip rereading xattrs and such in the initial code
-			entry = entry.LinkTo(targetEntry)
-			LOG_DEBUG("SNAPSHOT_UPLOAD", "Uploading cloned hardlink entry for %s to %s", entry.Path, entry.Link)
+
+			targetEntry := hardLinkTable[i].entry
+			var startChunk, endChunk int
+
+			if targetEntry.Size > 0 {
+				startChunk = hardLinkTable[i].startChunk - lastEndChunk
+				endChunk = targetEntry.EndChunk
+			}
+			entry = entry.HardLinkTo(targetEntry, startChunk, endChunk)
+
+			if targetEntry.Size > 0 {
+				lastEndChunk = hardLinkTable[i].startChunk + endChunk
+			}
+
+			LOG_DEBUG("SNAPSHOT_UPLOAD", "Uploading cloned hardlink for %s to %s (%v %v)", entry.Path, targetEntry.Path, startChunk, endChunk)
+		} else if entry.IsHardlinkRoot() {
+			hardLinkTable = append(hardLinkTable, hardLinkEntry{entry, 0})
 		}

 		buffer.Reset()
@@ -1229,9 +1297,10 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
 // Restore downloads a file from the storage.  If 'inPlace' is false, the download file is saved first to a temporary
 // file under the .duplicacy directory and then replaces the existing one.  Otherwise, the existing file will be
 // overwritten directly.
-// Return: true, nil:    Restored file; 
-//         false, nil:   Skipped file; 
-//         false, error: Failure to restore file (only if allowFailures == true)
+// Return: true, nil:    Restored file;
+//
+//	false, nil:   Skipped file;
+//	false, error: Failure to restore file (only if allowFailures == true)
 func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chunkMaker *ChunkMaker, entry *Entry, top string, inPlace bool, overwrite bool,
 	showStatistics bool, totalFileSize int64, downloadedFileSize int64, startTime int64, allowFailures bool) (bool, error) {

@@ -1280,6 +1349,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 					LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing: %v", fullPath, err)
 					return false, nil
 				}
+				entry.RestoreEarlyFileFlags(existingFile)

 				n := int64(1)
 				// There is a go bug on Windows (https://github.com/golang/go/issues/21681) that causes Seek to fail
@@ -1406,7 +1476,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			// fileHash != entry.Hash, warn/error depending on -overwrite option
 			if !overwrite && !isNewFile {
 				LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
-							"File %s already exists.  Please specify the -overwrite option to overwrite", entry.Path)
+					"File %s already exists.  Please specify the -overwrite option to overwrite", entry.Path)
 				return false, fmt.Errorf("file exists")
 			}

@@ -1463,6 +1533,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 				return false, nil
 			}
 		}
+		entry.RestoreEarlyFileFlags(existingFile)

 		existingFile.Seek(0, 0)

@@ -1545,6 +1616,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			LOG_ERROR("DOWNLOAD_OPEN", "Failed to open file for writing: %v", err)
 			return false, nil
 		}
+		entry.RestoreEarlyFileFlags(newFile)

 		hasher := manager.config.NewFileHasher()

@@ -1651,7 +1723,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho

 	if otherManager.config.DataShards != 0 && otherManager.config.ParityShards != 0 {
 		LOG_INFO("BACKUP_ERASURECODING", "Erasure coding is enabled for the destination storage with %d data shards and %d parity shards",
-		         otherManager.config.DataShards, otherManager.config.ParityShards)
+			otherManager.config.DataShards, otherManager.config.ParityShards)
 	}

 	if otherManager.config.rsaPublicKey != nil && len(otherManager.config.FileKey) > 0 {
@@ -1752,15 +1824,15 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 		LOG_TRACE("SNAPSHOT_COPY", "Copying snapshot %s at revision %d", snapshot.ID, snapshot.Revision)

 		for _, chunkHash := range snapshot.FileSequence {
-			chunks[chunkHash] = true  // The chunk is a snapshot chunk
+			chunks[chunkHash] = true // The chunk is a snapshot chunk
 		}

 		for _, chunkHash := range snapshot.ChunkSequence {
-			chunks[chunkHash] = true  // The chunk is a snapshot chunk
+			chunks[chunkHash] = true // The chunk is a snapshot chunk
 		}

 		for _, chunkHash := range snapshot.LengthSequence {
-			chunks[chunkHash] = true  // The chunk is a snapshot chunk
+			chunks[chunkHash] = true // The chunk is a snapshot chunk
 		}

 		description := manager.SnapshotManager.DownloadSequence(snapshot.ChunkSequence)
@@ -1773,7 +1845,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho

 		for _, chunkHash := range snapshot.ChunkHashes {
 			if _, found := chunks[chunkHash]; !found {
-				chunks[chunkHash] = false  // The chunk is a file chunk
+				chunks[chunkHash] = false // The chunk is a file chunk
 			}
 		}

@@ -1805,7 +1877,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 		}
 	}

-	LOG_INFO("SNAPSHOT_COPY", "Chunks to copy: %d, to skip: %d, total: %d", len(chunksToCopy), len(chunks) - len(chunksToCopy), len(chunks))
+	LOG_INFO("SNAPSHOT_COPY", "Chunks to copy: %d, to skip: %d, total: %d", len(chunksToCopy), len(chunks)-len(chunksToCopy), len(chunks))

 	chunkDownloader := CreateChunkOperator(manager.config, manager.storage, nil, false, false, downloadingThreads, false)

@@ -1814,7 +1886,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho

 	copiedChunks := 0
 	chunkUploader := CreateChunkOperator(otherManager.config, otherManager.storage, nil, false, false, uploadingThreads, false)
-	chunkUploader.UploadCompletionFunc =  func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
+	chunkUploader.UploadCompletionFunc = func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
 		action := "Skipped"
 		if !skipped {
 			copiedChunks++
@@ -1825,11 +1897,11 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho

 		elapsedTime := time.Now().Sub(startTime).Seconds()
 		speed := int64(float64(atomic.LoadInt64(&uploadedBytes)) / elapsedTime)
-		remainingTime := int64(float64(len(chunksToCopy) - chunkIndex - 1) / float64(chunkIndex + 1) * elapsedTime)
-		percentage := float64(chunkIndex + 1) / float64(len(chunksToCopy)) * 100.0
+		remainingTime := int64(float64(len(chunksToCopy)-chunkIndex-1) / float64(chunkIndex+1) * elapsedTime)
+		percentage := float64(chunkIndex+1) / float64(len(chunksToCopy)) * 100.0
 		LOG_INFO("COPY_PROGRESS", "%s chunk %s (%d/%d) %sB/s %s %.1f%%",
-				action, chunk.GetID(), chunkIndex + 1, len(chunksToCopy),
-				PrettySize(speed), PrettyTime(remainingTime), percentage)
+			action, chunk.GetID(), chunkIndex+1, len(chunksToCopy),
+			PrettySize(speed), PrettyTime(remainingTime), percentage)
 		otherManager.config.PutChunk(chunk)
 	}

@@ -1852,7 +1924,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 	chunkDownloader.Stop()
 	chunkUploader.Stop()

-	LOG_INFO("SNAPSHOT_COPY", "Copied %d new chunks and skipped %d existing chunks", copiedChunks, len(chunks) - copiedChunks)
+	LOG_INFO("SNAPSHOT_COPY", "Copied %d new chunks and skipped %d existing chunks", copiedChunks, len(chunks)-copiedChunks)

 	for _, snapshot := range snapshots {
 		if revisionMap[snapshot.ID][snapshot.Revision] == false {
--- a/src/duplicacy_entry.go
+++ b/src/duplicacy_entry.go
@@ -8,6 +8,7 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -23,6 +24,11 @@ import (
 	"github.com/vmihailenco/msgpack"
 )

+const (
+	entrySymHardLinkRootChunkMarker   = -72
+	entrySymHardLinkTargetChunkMarker = -73
+)
+
 // This is the hidden directory in the repository for storing various files.
 var DUPLICACY_DIRECTORY = ".duplicacy"
 var DUPLICACY_FILE = ".duplicacy"
@@ -119,7 +125,7 @@ func (entry *Entry) Copy() *Entry {
 	}
 }

-func (entry *Entry) LinkTo(target *Entry) *Entry {
+func (entry *Entry) HardLinkTo(target *Entry, startChunk int, endChunk int) *Entry {
 	return &Entry{
 		Path: entry.Path,
 		Size: target.Size,
@@ -131,9 +137,9 @@ func (entry *Entry) LinkTo(target *Entry) *Entry {
 		UID: target.UID,
 		GID: target.GID,

-		StartChunk:  target.StartChunk,
+		StartChunk:  startChunk,
 		StartOffset: target.StartOffset,
-		EndChunk:    target.EndChunk,
+		EndChunk:    endChunk,
 		EndOffset:   target.EndOffset,

 		Attributes: target.Attributes,
@@ -509,20 +515,36 @@ func (entry *Entry) IsLink() bool {
 	return entry.Mode&uint32(os.ModeSymlink) != 0
 }

+func (entry *Entry) IsSpecial() bool {
+	return entry.Mode&uint32(os.ModeNamedPipe|os.ModeDevice|os.ModeCharDevice|os.ModeSocket) != 0
+}
+
+func (entry *Entry) IsFileOrSpecial() bool {
+	return entry.Mode&uint32(os.ModeDir|os.ModeSymlink|os.ModeIrregular) == 0
+}
+
 func (entry *Entry) IsComplete() bool {
 	return entry.Size >= 0
 }

-func (entry *Entry) IsFileNotHardlink() bool {
-	return entry.IsFile() && (len(entry.Link) == 0 || entry.Link == "/")
-}
-
 func (entry *Entry) IsHardlinkedFrom() bool {
-	return entry.IsFile() && len(entry.Link) > 0 && entry.Link != "/"
+	return (entry.IsFileOrSpecial() && len(entry.Link) > 0 && entry.Link != "/") || (entry.IsLink() && entry.StartChunk == entrySymHardLinkTargetChunkMarker)
 }

 func (entry *Entry) IsHardlinkRoot() bool {
-	return entry.IsFile() && entry.Link == "/"
+	return (entry.IsFileOrSpecial() && entry.Link == "/") || (entry.IsLink() && entry.StartChunk == entrySymHardLinkRootChunkMarker)
+}
+
+func (entry *Entry) GetHardlinkId() (int, error) {
+	if entry.IsLink() {
+		if entry.StartChunk != entrySymHardLinkTargetChunkMarker {
+			return 0, errors.New("Symlink entry not marked as hardlinked")
+		}
+		return entry.StartOffset, nil
+	} else {
+		i, err := strconv.ParseUint(entry.Link, 16, 64)
+		return int(i), err
+	}
 }

 func (entry *Entry) GetPermissions() os.FileMode {
@@ -583,6 +605,10 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setO
 		}
 	}

+	if entry.Attributes != nil && len(*entry.Attributes) > 0 {
+		entry.SetAttributesToFile(fullPath)
+	}
+
 	// Only set the time if the file is not a symlink
 	if !entry.IsLink() && (*fileInfo).ModTime().Unix() != entry.Time {
 		modifiedTime := time.Unix(entry.Time, 0)
@@ -593,10 +619,6 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setO
 		}
 	}

-	if entry.Attributes != nil && len(*entry.Attributes) > 0 {
-		entry.SetAttributesToFile(fullPath)
-	}
-
 	return true
 }

@@ -733,12 +755,13 @@ type listEntryLinkKey struct {
 }

 type ListingState struct {
-	linkTable map[listEntryLinkKey]string // map unique inode details to initially found path
+	linkIndex int
+	linkTable map[listEntryLinkKey]int // map unique inode details to initially found path
 }

 func NewListingState() *ListingState {
 	return &ListingState{
-		linkTable: make(map[listEntryLinkKey]string),
+		linkTable: make(map[listEntryLinkKey]int),
 	}
 }

@@ -784,10 +807,41 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
 		if f.Name() == DUPLICACY_DIRECTORY {
 			continue
 		}
+
 		entry := CreateEntryFromFileInfo(f, normalizedPath)
 		if len(patterns) > 0 && !MatchPath(entry.Path, patterns) {
 			continue
 		}
+
+		var linkKey *listEntryLinkKey
+
+		if runtime.GOOS != "windows" && !entry.IsDir() {
+			if stat := f.Sys().(*syscall.Stat_t); stat != nil && stat.Nlink > 1 {
+				k := listEntryLinkKey{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}
+				if linkIndex, seen := listingState.linkTable[k]; seen {
+					if linkIndex == -1 {
+						LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute (hardlink)", entry.Path)
+						continue
+					}
+					entry.Size = 0
+					if entry.IsLink() {
+						entry.StartChunk = entrySymHardLinkTargetChunkMarker
+						entry.StartOffset = linkIndex
+					} else {
+						entry.Link = strconv.FormatInt(int64(linkIndex), 16)
+					}
+				} else {
+					if entry.IsLink() {
+						entry.StartChunk = entrySymHardLinkRootChunkMarker
+					} else {
+						entry.Link = "/"
+					}
+					listingState.linkTable[k] = -1
+					linkKey = &k
+				}
+			}
+		}
+
 		if entry.IsLink() {
 			isRegular := false
 			isRegular, entry.Link, err = Readlink(joinPath(top, entry.Path))
@@ -818,6 +872,12 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
 				}
 				entry = newEntry
 			}
+		} else if entry.IsSpecial() {
+			if !entry.ReadSpecial(f) {
+				LOG_WARN("LIST_DEV", "Failed to save device node %s", entry.Path)
+				skippedFiles = append(skippedFiles, entry.Path)
+				continue
+			}
 		}

 		entry.ReadAttributes(top)
@@ -827,24 +887,9 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
 			continue
 		}

-		if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
-			LOG_WARN("LIST_SKIP", "Skipped non-regular file %s", entry.Path)
-			skippedFiles = append(skippedFiles, entry.Path)
-			continue
-		}
-
-		if entry.IsFile() {
-			stat, ok := f.Sys().(*syscall.Stat_t)
-			if ok && stat != nil && stat.Nlink > 1 {
-				k := listEntryLinkKey{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}
-				if path, ok := listingState.linkTable[k]; ok {
-					LOG_DEBUG("LIST_HARDLINK", "Detected hardlink %s to %s", entry.Path, path)
-					entry.Link = path
-				} else {
-					entry.Link = "/"
-					listingState.linkTable[k] = entry.Path
-				}
-			}
+		if linkKey != nil {
+			listingState.linkTable[*linkKey] = listingState.linkIndex
+			listingState.linkIndex++
 		}

 		if entry.IsDir() {
--- a/src/duplicacy_entrylist.go
+++ b/src/duplicacy_entrylist.go
@@ -62,7 +62,6 @@ type EntryList struct {
 	uploadedChunkIndex int                  // counter for upload chunks
 	uploadedChunkOffset int                 // the start offset for the current modified entry

-	HardLinkTable map[string]*Entry
 }

 // Create a new entry list
@@ -79,7 +78,6 @@ func CreateEntryList(snapshotID string, cachePath string, maximumInMemoryEntries
 		maximumInMemoryEntries: maximumInMemoryEntries,
 		cachePath: cachePath,
 		Token: string(token),
-		HardLinkTable: make(map[string]*Entry),
 	}

 	return entryList, nil
@@ -113,14 +111,14 @@ func (entryList *EntryList)createOnDiskFile() error {
 // Add an entry to the entry list
 func (entryList *EntryList)AddEntry(entry *Entry) error {

-	if !entry.IsDir() && !entry.IsLink() {
+	if entry.IsFile() {
 		entryList.NumberOfEntries++
 	}

 	if !entry.IsComplete() {
-		if entry.IsDir() || entry.IsLink() {
+		if !entry.IsFile() {
 			entry.Size = 0
-		} else if !entry.IsHardlinkedFrom() {
+		} else {
 			modifiedEntry := ModifiedEntry {
 				Path: entry.Path,
 				Size: -1,
--- a/src/duplicacy_snapshot.go
+++ b/src/duplicacy_snapshot.go
@@ -68,7 +68,7 @@ func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
 								         skippedDirectories *[]string, skippedFiles *[]string) {

 	var patterns []string
-	var listingState = NewListingState()
+	listingState := NewListingState()

 	if filtersFile == "" {
 		filtersFile = joinPath(GetDuplicacyPreferencePath(), "filters")
--- a/src/duplicacy_storage.go
+++ b/src/duplicacy_storage.go
@@ -756,6 +756,8 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the Storj storage at %s: %v", storageURL, err)
 			return nil
 		}
+		SavePassword(preference, "storj_key", apiKey)
+		SavePassword(preference, "storj_passphrase", passphrase)
 		return storjStorage
 	} else if matched[1] == "smb" {
 		server := matched[3]
--- a/src/duplicacy_utils.go
+++ b/src/duplicacy_utils.go
@@ -7,14 +7,15 @@ package duplicacy
 import (
 	"bufio"
 	"crypto/sha256"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
-	"runtime"

 	"github.com/gilbertchen/gopass"
 	"golang.org/x/crypto/pbkdf2"
@@ -56,7 +57,7 @@ func IsEmptyFilter(pattern string) bool {
 }

 func IsUnspecifiedFilter(pattern string) bool {
-	if pattern[0] != '+' && pattern[0] != '-' && !strings.HasPrefix(pattern, "i:") && !strings.HasPrefix(pattern, "e:")  {
+	if pattern[0] != '+' && pattern[0] != '-' && !strings.HasPrefix(pattern, "i:") && !strings.HasPrefix(pattern, "e:") {
 		return true
 	} else {
 		return false
@@ -275,7 +276,6 @@ func SavePassword(preference Preference, passwordType string, password string) {
 // The following code was modified from the online article  'Matching Wildcards: An Algorithm', by Kirk J. Krauss,
 // Dr. Dobb's, August 26, 2008. However, the version in the article doesn't handle cases like matching 'abcccd'
 // against '*ccd', and the version here fixed that issue.
-//
 func matchPattern(text string, pattern string) bool {

 	textLength := len(text)
@@ -469,8 +469,39 @@ func PrintMemoryUsage() {
 		runtime.ReadMemStats(&m)

 		LOG_INFO("MEMORY_STATS", "Currently allocated: %s, total allocated: %s, system memory: %s, number of GCs: %d",
-		         PrettySize(int64(m.Alloc)), PrettySize(int64(m.TotalAlloc)), PrettySize(int64(m.Sys)), m.NumGC)
+			PrettySize(int64(m.Alloc)), PrettySize(int64(m.TotalAlloc)), PrettySize(int64(m.Sys)), m.NumGC)

 		time.Sleep(time.Second)
 	}
-}
+}
+
+func (entry *Entry) dump() map[string]interface{} {
+
+	object := make(map[string]interface{})
+
+	object["path"] = entry.Path
+	object["size"] = entry.Size
+	object["time"] = entry.Time
+	object["mode"] = entry.Mode
+	object["hash"] = entry.Hash
+	object["link"] = entry.Link
+
+	object["content"] = fmt.Sprintf("%d:%d:%d:%d",
+		entry.StartChunk, entry.StartOffset, entry.EndChunk, entry.EndOffset)
+
+	if entry.UID != -1 && entry.GID != -1 {
+		object["uid"] = entry.UID
+		object["gid"] = entry.GID
+	}
+
+	if entry.Attributes != nil && len(*entry.Attributes) > 0 {
+		object["attributes"] = entry.Attributes
+	}
+
+	return object
+}
+
+func (entry *Entry) dumpString() string {
+	data, _ := json.Marshal(entry.dump())
+	return string(data)
+}
--- a/src/duplicacy_utils_bsd_common.go
+++ b/src/duplicacy_utils_bsd_common.go
@@ -0,0 +1,109 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+//go:build freebsd || netbsd || darwin
+// +build freebsd netbsd darwin
+
+package duplicacy
+
+import (
+	"bytes"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/pkg/xattr"
+)
+
+const bsdFileFlagsKey = "\x00bf"
+
+func (entry *Entry) ReadAttributes(top string) {
+	fullPath := filepath.Join(top, entry.Path)
+	fileInfo, err := os.Lstat(fullPath)
+	if err != nil {
+		return
+	}
+
+	if !entry.IsSpecial() {
+		attributes, _ := xattr.LList(fullPath)
+		if len(attributes) > 0 {
+			entry.Attributes = &map[string][]byte{}
+			for _, name := range attributes {
+				attribute, err := xattr.LGet(fullPath, name)
+				if err == nil {
+					(*entry.Attributes)[name] = attribute
+				}
+			}
+		}
+	}
+	if err := entry.readFileFlags(fileInfo); err != nil {
+		LOG_INFO("ATTR_BACKUP", "Could not backup flags for file %s: %v", fullPath, err)
+	}
+}
+
+func (entry *Entry) SetAttributesToFile(fullPath string) {
+	if !entry.IsSpecial() {
+		names, _ := xattr.LList(fullPath)
+		for _, name := range names {
+			newAttribute, found := (*entry.Attributes)[name]
+			if found {
+				oldAttribute, _ := xattr.LGet(fullPath, name)
+				if !bytes.Equal(oldAttribute, newAttribute) {
+					xattr.LSet(fullPath, name, newAttribute)
+				}
+				delete(*entry.Attributes, name)
+			} else {
+				xattr.LRemove(fullPath, name)
+			}
+		}
+
+		for name, attribute := range *entry.Attributes {
+			if len(name) > 0 && name[0] == '\x00' {
+				continue
+			}
+			xattr.LSet(fullPath, name, attribute)
+		}
+	}
+	if err := entry.restoreLateFileFlags(fullPath); err != nil {
+		LOG_DEBUG("ATTR_RESTORE", "Could not restore flags for file %s: %v", fullPath, err)
+	}
+}
+
+func (entry *Entry) readFileFlags(fileInfo os.FileInfo) error {
+	stat, ok := fileInfo.Sys().(*syscall.Stat_t)
+	if ok && stat.Flags != 0 {
+		if entry.Attributes == nil {
+			entry.Attributes = &map[string][]byte{}
+		}
+		v := make([]byte, 4)
+		binary.LittleEndian.PutUint32(v, stat.Flags)
+		(*entry.Attributes)[bsdFileFlagsKey] = v
+		LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", stat.Flags, entry.Path)
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyDirFlags(path string) error {
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
+	return nil
+}
+
+func (entry *Entry) RestoreSpecial(fullPath string) error {
+	mode := entry.Mode & uint32(fileModeMask)
+
+	if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
+		mode |= syscall.S_IFIFO
+	} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
+		mode |= syscall.S_IFCHR
+	} else if entry.Mode&uint32(os.ModeDevice) != 0 {
+		mode |= syscall.S_IFBLK
+	} else {
+		return nil
+	}
+	return syscall.Mknod(fullPath, mode, int(entry.GetRdev()))
+}
--- a/src/duplicacy_utils_darwin.go
+++ b/src/duplicacy_utils_darwin.go
@@ -5,10 +5,29 @@
 package duplicacy

 import (
+	"encoding/binary"
+	"os"
 	"strings"
+	"syscall"
 )

 func excludedByAttribute(attributes map[string][]byte) bool {
 	value, ok := attributes["com.apple.metadata:com_apple_backup_excludeItem"]
 	return ok && strings.Contains(string(value), "com.apple.backupd")
 }
+
+func (entry *Entry) restoreLateFileFlags(path string) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
+		f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_SYMLINK, 0)
+		if err != nil {
+			return err
+		}
+		err = syscall.Fchflags(int(f.Fd()), int(binary.LittleEndian.Uint32(v)))
+		f.Close()
+		return err
+	}
+	return nil
+}
--- a/src/duplicacy_utils_linux.go
+++ b/src/duplicacy_utils_linux.go
@@ -0,0 +1,242 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+	"bytes"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+	"syscall"
+	"unsafe"
+
+	"github.com/pkg/xattr"
+)
+
+const (
+	linux_FS_SECRM_FL        = 0x00000001 /* Secure deletion */
+	linux_FS_UNRM_FL         = 0x00000002 /* Undelete */
+	linux_FS_COMPR_FL        = 0x00000004 /* Compress file */
+	linux_FS_SYNC_FL         = 0x00000008 /* Synchronous updates */
+	linux_FS_IMMUTABLE_FL    = 0x00000010 /* Immutable file */
+	linux_FS_APPEND_FL       = 0x00000020 /* writes to file may only append */
+	linux_FS_NODUMP_FL       = 0x00000040 /* do not dump file */
+	linux_FS_NOATIME_FL      = 0x00000080 /* do not update atime */
+	linux_FS_NOCOMP_FL       = 0x00000400 /* Don't compress */
+	linux_FS_JOURNAL_DATA_FL = 0x00004000 /* Reserved for ext3 */
+	linux_FS_NOTAIL_FL       = 0x00008000 /* file tail should not be merged */
+	linux_FS_DIRSYNC_FL      = 0x00010000 /* dirsync behaviour (directories only) */
+	linux_FS_TOPDIR_FL       = 0x00020000 /* Top of directory hierarchies*/
+	linux_FS_NOCOW_FL        = 0x00800000 /* Do not cow file */
+	linux_FS_PROJINHERIT_FL  = 0x20000000 /* Create with parents projid */
+
+	linux_FS_IOC_GETFLAGS uintptr = 0x80086601
+	linux_FS_IOC_SETFLAGS uintptr = 0x40086602
+
+	linuxIocFlagsFileEarly = linux_FS_SECRM_FL | linux_FS_UNRM_FL | linux_FS_COMPR_FL | linux_FS_NODUMP_FL | linux_FS_NOATIME_FL | linux_FS_NOCOMP_FL | linux_FS_JOURNAL_DATA_FL | linux_FS_NOTAIL_FL | linux_FS_NOCOW_FL
+	linuxIocFlagsDirEarly  = linux_FS_TOPDIR_FL | linux_FS_PROJINHERIT_FL
+	linuxIocFlagsLate      = linux_FS_SYNC_FL | linux_FS_IMMUTABLE_FL | linux_FS_APPEND_FL | linux_FS_DIRSYNC_FL
+
+	linuxFileFlagsKey = "\x00lf"
+)
+
+func ioctl(f *os.File, request uintptr, attrp *uint32) error {
+	argp := uintptr(unsafe.Pointer(attrp))
+
+	if _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), request, argp); errno != 0 {
+		return os.NewSyscallError("ioctl", errno)
+	}
+	return nil
+}
+
+type xattrHandle struct {
+	f        *os.File
+	fullPath string
+}
+
+func (x xattrHandle) list() ([]string, error) {
+	if x.f != nil {
+		return xattr.FList(x.f)
+	} else {
+		return xattr.LList(x.fullPath)
+	}
+}
+
+func (x xattrHandle) get(name string) ([]byte, error) {
+	if x.f != nil {
+		return xattr.FGet(x.f, name)
+	} else {
+		return xattr.LGet(x.fullPath, name)
+	}
+}
+
+func (x xattrHandle) set(name string, value []byte) error {
+	if x.f != nil {
+		return xattr.FSet(x.f, name, value)
+	} else {
+		return xattr.LSet(x.fullPath, name, value)
+	}
+}
+
+func (x xattrHandle) remove(name string) error {
+	if x.f != nil {
+		return xattr.FRemove(x.f, name)
+	} else {
+		return xattr.LRemove(x.fullPath, name)
+	}
+}
+
+func (entry *Entry) ReadAttributes(top string) {
+	fullPath := filepath.Join(top, entry.Path)
+	x := xattrHandle{nil, fullPath}
+
+	if !entry.IsLink() {
+		var err error
+		x.f, err = os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
+		if err != nil {
+			// FIXME: We really should return errors for failure to read
+			return
+		}
+	}
+
+	attributes, _ := x.list()
+
+	if len(attributes) > 0 {
+		entry.Attributes = &map[string][]byte{}
+	}
+	for _, name := range attributes {
+		attribute, err := x.get(name)
+		if err == nil {
+			(*entry.Attributes)[name] = attribute
+		}
+	}
+
+	if entry.IsFile() || entry.IsDir() {
+		if err := entry.readFileFlags(x.f); err != nil {
+			LOG_INFO("ATTR_BACKUP", "Could not backup flags for file %s: %v", fullPath, err)
+		}
+	}
+	x.f.Close()
+}
+
+func (entry *Entry) SetAttributesToFile(fullPath string) {
+	x := xattrHandle{nil, fullPath}
+	if !entry.IsLink() {
+		var err error
+		x.f, err = os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
+		if err != nil {
+			return
+		}
+	}
+
+	names, _ := x.list()
+
+	for _, name := range names {
+		newAttribute, found := (*entry.Attributes)[name]
+		if found {
+			oldAttribute, _ := x.get(name)
+			if !bytes.Equal(oldAttribute, newAttribute) {
+				x.set(name, newAttribute)
+			}
+			delete(*entry.Attributes, name)
+		} else {
+			x.remove(name)
+		}
+	}
+
+	for name, attribute := range *entry.Attributes {
+		if len(name) > 0 && name[0] == '\x00' {
+			continue
+		}
+		x.set(name, attribute)
+	}
+	if entry.IsFile() || entry.IsDir() {
+		if err := entry.restoreLateFileFlags(x.f); err != nil {
+			LOG_DEBUG("ATTR_RESTORE", "Could not restore flags for file %s: %v", fullPath, err)
+		}
+	}
+	x.f.Close()
+}
+
+func (entry *Entry) readFileFlags(f *os.File) error {
+	var flags uint32
+	if err := ioctl(f, linux_FS_IOC_GETFLAGS, &flags); err != nil {
+		return err
+	}
+	if flags != 0 {
+		if entry.Attributes == nil {
+			entry.Attributes = &map[string][]byte{}
+		}
+		v := make([]byte, 4)
+		binary.LittleEndian.PutUint32(v, flags)
+		(*entry.Attributes)[linuxFileFlagsKey] = v
+		LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", flags, entry.Path)
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyDirFlags(path string) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
+		flags := binary.LittleEndian.Uint32(v) & linuxIocFlagsDirEarly
+		f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_DIRECTORY, 0)
+		if err != nil {
+			return err
+		}
+		LOG_DEBUG("ATTR_RESTORE", "Restore dir flags (early) 0x%x for %s", flags, entry.Path)
+		err = ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
+		f.Close()
+		return err
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
+		flags := binary.LittleEndian.Uint32(v) & linuxIocFlagsFileEarly
+		LOG_DEBUG("ATTR_RESTORE", "Restore flags (early) 0x%x for %s", flags, entry.Path)
+		return ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
+	}
+	return nil
+}
+
+func (entry *Entry) restoreLateFileFlags(f *os.File) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
+		flags := binary.LittleEndian.Uint32(v) & (linuxIocFlagsFileEarly | linuxIocFlagsDirEarly | linuxIocFlagsLate)
+		LOG_DEBUG("ATTR_RESTORE", "Restore flags (late) 0x%x for %s", flags, entry.Path)
+		return ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreSpecial(fullPath string) error {
+	mode := entry.Mode & uint32(fileModeMask)
+
+	if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
+		mode |= syscall.S_IFIFO
+	} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
+		mode |= syscall.S_IFCHR
+	} else if entry.Mode&uint32(os.ModeDevice) != 0 {
+		mode |= syscall.S_IFBLK
+	} else if entry.Mode&uint32(os.ModeSocket) != 0 {
+		mode |= syscall.S_IFSOCK
+	} else {
+		return nil
+	}
+	return syscall.Mknod(fullPath, mode, int(entry.GetRdev()))
+}
+
+func excludedByAttribute(attributes map[string][]byte) bool {
+	_, ok := attributes["user.duplicacy_exclude"]
+	return ok
+}
--- a/src/duplicacy_utils_others.go
+++ b/src/duplicacy_utils_others.go
@@ -2,18 +2,17 @@
 // Free for personal use and commercial trial
 // Commercial use requires per-user licenses available from https://duplicacy.com

+//go:build !windows
 // +build !windows

 package duplicacy

 import (
-	"bytes"
 	"os"
 	"path"
-	"path/filepath"
 	"syscall"

-	"github.com/pkg/xattr"
+	"golang.org/x/sys/unix"
 )

 func Readlink(path string) (isRegular bool, s string, err error) {
@@ -47,43 +46,35 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
 	return true
 }

-func (entry *Entry) ReadAttributes(top string) {
-
-	fullPath := filepath.Join(top, entry.Path)
-	attributes, _ := xattr.List(fullPath)
-	if len(attributes) > 0 {
-		entry.Attributes = &map[string][]byte{}
-		for _, name := range attributes {
-			attribute, err := xattr.Get(fullPath, name)
-			if err == nil {
-				(*entry.Attributes)[name] = attribute
-			}
-		}
+func (entry *Entry) ReadSpecial(fileInfo os.FileInfo) bool {
+	if fileInfo.Mode()&(os.ModeDevice|os.ModeCharDevice) == 0 {
+		return true
 	}
+	stat := fileInfo.Sys().(*syscall.Stat_t)
+	if stat == nil {
+		return false
+	}
+	entry.Size = 0
+	rdev := uint64(stat.Rdev)
+	entry.StartChunk = int(rdev & 0xFFFFFFFF)
+	entry.StartOffset = int(rdev >> 32)
+	return true
 }

-func (entry *Entry) SetAttributesToFile(fullPath string) {
-	names, _ := xattr.List(fullPath)
+func (entry *Entry) GetRdev() uint64 {
+	return uint64(entry.StartChunk) | uint64(entry.StartOffset)<<32
+}

-	for _, name := range names {
-
-
-		newAttribute, found := (*entry.Attributes)[name]
-		if found {
-			oldAttribute, _ := xattr.Get(fullPath, name)
-			if !bytes.Equal(oldAttribute, newAttribute) {
-				xattr.Set(fullPath, name, newAttribute)
-			}
-			delete(*entry.Attributes, name)
-		} else {
-			xattr.Remove(fullPath, name)
-		}
-	}
-
-	for name, attribute := range *entry.Attributes {
-		xattr.Set(fullPath, name, attribute)
+func (entry *Entry) IsSameSpecial(fileInfo os.FileInfo) bool {
+	stat := fileInfo.Sys().(*syscall.Stat_t)
+	if stat == nil {
+		return false
 	}
+	return (uint32(fileInfo.Mode()) == entry.Mode) && (uint64(stat.Rdev) == entry.GetRdev())
+}

+func MakeHardlink(source string, target string) error {
+	return unix.Linkat(unix.AT_FDCWD, source, unix.AT_FDCWD, target, 0)
 }

 func joinPath(components ...string) string {
--- a/src/duplicacy_utils_posix.go
+++ b/src/duplicacy_utils_posix.go
@@ -2,8 +2,8 @@
 // Free for personal use and commercial trial
 // Commercial use requires per-user licenses available from https://duplicacy.com

-//go:build freebsd || netbsd || linux || solaris
-// +build freebsd netbsd linux solaris
+//go:build freebsd || netbsd || solaris
+// +build freebsd netbsd solaris

 package duplicacy

--- a/src/duplicacy_utils_windows.go
+++ b/src/duplicacy_utils_windows.go
@@ -117,6 +117,18 @@ func (entry *Entry) SetAttributesToFile(fullPath string) {

 }

+func (entry *Entry) ReadDeviceNode(fileInfo os.FileInfo) bool {
+	return nil
+}
+
+func (entry *Entry) RestoreSpecial(fullPath string) error {
+	return nil
+}
+
+func MakeHardlink(source string, target string) error {
+	return os.Link(source, target)
+}
+
 func joinPath(components ...string) string {

 	combinedPath := `\\?\` + filepath.Join(components...)
@@ -132,6 +144,18 @@ func SplitDir(fullPath string) (dir string, file string) {
 	return fullPath[:i+1], fullPath[i+1:]
 }

-func excludedByAttribute(attributes map[string][]byte) bool {
-	return false
+func (entry *Entry) ReadFileFlags(f *os.File) error {
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyDirFlags(path string) error {
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
+	return nil
+}
+
+func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
+	return nil
 }
--- a/src/duplicacy_utils_xbsd.go
+++ b/src/duplicacy_utils_xbsd.go
@@ -0,0 +1,30 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+//go:build freebsd || netbsd
+// +build freebsd netbsd
+
+package duplicacy
+
+import (
+	"bytes"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/pkg/xattr"
+)
+
+func (entry *Entry) restoreLateFileFlags(path string) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
+		if _, _, errno := syscall.Syscall(syscall.SYS_LCHFLAGS, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(v), 0); errno != 0 {
+			return os.NewSyscallError("lchflags", errno)
+		}
+	}
+	return nil
+}
Author	SHA1	Message	Date
John K. Luebs	28efe91c3f	Support backup of sockets This only works on Linux. Darwin does not allow mknod of a socket. Have not tested BSD.	2023-10-04 02:53:48 -05:00
John K. Luebs	73ca9794ab	gofmt some files	2023-10-04 02:53:48 -05:00
John K. Luebs	70ea4d3acf	Fix handling of hardlinks and special files Also, don't attempt xattr ops on special files on the BSD likes. TODO: Should have a way to allow restore without special files, otherwise very cumbersome for a regular user.	2023-10-04 02:53:48 -05:00
John K. Luebs	a1a3f3d4cb	Support for hardlinks to symlinks This is a specialized use-case, but it is indeed possible to do this and can be used if xattrs are attached to the symlink.	2023-10-04 02:53:48 -05:00
John K. Luebs	5b40bf3d93	Fix handling of xattrs with symlinks Fix Linux, Darwin, and other BSD (untested) to allow proper handling of xattrs with symlinks. On Linux we cannot use the f* syscalls for symlinks because symlinks cannot be opened. File flags must be handled differently on darwin and other BSD due to the lack of the LCHFLAGS syscall on darwin, and the fact that it is emulated in libc. However, we do have O_SYMLINK on darwin.	2023-10-04 02:53:48 -05:00
John K. Luebs	5087ac738d	Support backup and restore of special files on POSIX style systems Special files are device nodes and named pipes. The necessity of the former is clear, the latter is debatable. In order to preserve backward compatibility, the device number is encoded in the StartChunk/StartOffset fields of the entry.	2023-10-04 02:53:48 -05:00
John K. Luebs	99e4dcae00	Don't overwrite symlinks if file already exists	2023-10-04 02:53:48 -05:00
John K. Luebs	997ff5bdf6	Support backup and restore of hardlinks This tracks inode/device from the stat info and creates backward compatible snapshots that allow preserving hardlinks. Backwards compatibility is preserved by saving a virtual inode number index in the Link field of the file entry. Since this field was previously only used for symlinks, this won't break old versions. Additionally, the entry data is cloned so restoration with an old version works. Current limitations are primarility with restore. They include: - no command line option to prevent hard link restore - if a file has the immutable or append only flag it will be set before hardlinks are restored, so hardlinking will fail. - if a partial restore includes a hardlink but not the parent directories the hardlink will fail. These will be solved by grouping restore of hardlinks together with file, prior to applying final metadata. - if a file is changed and is being rewritten by a restore hardlinks are not preserved.	2023-10-04 02:53:48 -05:00
John K. Luebs	34b1e19278	Initial implementation of file/inode flags (Linux, BSD, darwin) Basic support for BSD and Darwin style chflags (stat flags). Applies these flags at the end of file restore. Supports linux style ioctl_iflags(2) in a 2 step process. Flags that need to be applied prior to writes such as compress and especially no-COW are applied immediately upon file open. The flags format is backwards compatible. An attribute starting with a null byte is used to store flags in the entry attributes table. With an old version of duplicacy the restore of this attribute should silently fail (effectively be ignored). Fixes xattr restore to use O_NOFOLLOW so attributes are applied to symlink. TODO: Tests, possible option to switch off mutable/append prior to restore of existing file similar to rsync. Does not apply attributes or flags to the top most directory.	2023-10-04 02:53:48 -05:00
John K. Luebs	c1c8af1de9	Increase b2 client max file listing count to 10000 Considerable speed improvement with listing large storage.	2023-10-04 02:53:48 -05:00
John K. Luebs	8b788572c8	Fix exclude_by_attribute feature on POSIX The exclude by attribute function is broken on non-Darwin POSIX: linux and freebsd. This is because those xattrs must be prefixed by a legal namespace. The old xattr library implicitly appended the user namespace to the xattr, but the current official go pkg does not (which is just as well). Also fix the test to remove the discordant old xattr dependency and provide test cases for both darwin and non-darwin POSIX.	2023-10-04 02:53:48 -05:00
John K. Luebs	b592484b54	Use S3 ListObjectsV2 for listing files ListObjects has been deprecated since 2016 and ListObjectsV2 with use of explicit pagination tokens is more performant for large listings as well. This also mitigates an issue with iDrive E2 where the StartAfter/Marker is included in the output, leading to duplicate entries. Right now this causes an exhaustive prune to delete chunks erroneously flagged as duplicate, destroying the storage.	2023-10-04 02:53:48 -05:00
Gilbert Chen	50120146df	Bump version to 3.2.2	2023-10-03 22:35:18 -04:00
Gilbert Chen	7bfc0e7d51	Save passwords after a storj storage has been created	2023-10-03 22:33:41 -04:00