Rename hardlink to hard link (two words)

This moves special file link data to the chunk fields like symlink
hardlinks.

duplicacy/duplicacy_main.go

@@ -2262,7 +2262,7 @@ func main() {
 	app.Name = "duplicacy"
 	app.HelpName = "duplicacy"
 	app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
-	app.Version = "3.2.1" + " (" + GitCommit + ")"
+	app.Version = "3.2.2" + " (" + GitCommit + ")"
 
 	// Exit with code 2 if an invalid command is provided
 	app.CommandNotFound = func(context *cli.Context, command string) {

go.mod

@@ -27,6 +27,7 @@ require (
 	golang.org/x/crypto v0.12.0
 	golang.org/x/net v0.10.0
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+	golang.org/x/sys v0.11.0
 	google.golang.org/api v0.21.0
 	storj.io/uplink v1.12.0
 )
@@ -63,7 +64,6 @@ require (
 	go.opencensus.io v0.22.3 // indirect
 	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/term v0.11.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/tools v0.9.1 // indirect

src/duplicacy_backupmanager.go

@@ -710,23 +710,23 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 	var hardLinkTable []hardLinkEntry
 	var hardLinks []*Entry
 
-	restoreHardlink := func(entry *Entry, fullPath string) bool {
+	restoreHardLink := func(entry *Entry, fullPath string) bool {
-		if entry.IsHardlinkRoot() {
+		if entry.IsHardLinkRoot() {
 			hardLinkTable[len(hardLinkTable)-1].willExist = true
-		} else if entry.IsHardlinkedFrom() {
+		} else if entry.IsHardLinkChild() {
-			i, err := entry.GetHardlinkId()
+			i, err := entry.GetHardLinkId()
 			if err != nil {
-				LOG_ERROR("RESTORE_HARDLINK", "Decode error for hardlinked entry %s, %v", entry.Path, err)
+				LOG_ERROR("RESTORE_HARDLINK", "Decode error for hard link entry %s: %v", entry.Path, err)
 				return false
 			}
 			if !hardLinkTable[i].willExist {
 				hardLinkTable[i] = hardLinkEntry{entry, true}
 			} else {
 				sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
-				LOG_INFO("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath)
 				if err := MakeHardlink(sourcePath, fullPath); err != nil {
-					LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s %v", fullPath, sourcePath, err)
+					LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s: %v", fullPath, sourcePath, err)
 				}
+				LOG_TRACE("DOWNLOAD_DONE", "Hard linked %s to %s", entry.Path, hardLinkTable[i].entry.Path)
 				return true
 			}
 		}
@@ -735,7 +735,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 
 	for remoteEntry := range remoteListingChannel {
 
-		if remoteEntry.IsHardlinkRoot() {
+		if remoteEntry.IsHardLinkRoot() {
 			hardLinkTable = append(hardLinkTable, hardLinkEntry{remoteEntry, false})
 		}
 
@@ -782,7 +782,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 					isRegular, link, err := Readlink(fullPath)
 					if err == nil && link == remoteEntry.Link && !isRegular {
 						remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
-						if remoteEntry.IsHardlinkRoot() {
+						if remoteEntry.IsHardLinkRoot() {
 							hardLinkTable[len(hardLinkTable)-1].willExist = true
 						}
 						continue
@@ -798,7 +798,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 				os.Remove(fullPath)
 			}
 
-			if restoreHardlink(remoteEntry, fullPath) {
+			if restoreHardLink(remoteEntry, fullPath) {
 				continue
 			}
 
@@ -807,8 +807,8 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 				return 0
 			}
 			remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
-
 			LOG_TRACE("DOWNLOAD_DONE", "Symlink %s updated", remoteEntry.Path)
+
 		} else if remoteEntry.IsDir() {
 
 			stat, err := os.Stat(fullPath)
@@ -833,7 +833,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 			if stat, _ := os.Lstat(fullPath); stat != nil {
 				if remoteEntry.IsSameSpecial(stat) {
 					remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
-					if remoteEntry.IsHardlinkRoot() {
+					if remoteEntry.IsHardLinkRoot() {
 						hardLinkTable[len(hardLinkTable)-1].willExist = true
 					}
 				}
@@ -845,22 +845,24 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 				os.Remove(fullPath)
 			}
 
-			if restoreHardlink(remoteEntry, fullPath) {
+			if restoreHardLink(remoteEntry, fullPath) {
 				continue
 			}
 
 			if err := remoteEntry.RestoreSpecial(fullPath); err != nil {
-				LOG_ERROR("RESTORE_SPECIAL", "Unable to restore special file %s: %v", remoteEntry.Path, err)
+				LOG_ERROR("RESTORE_SPECIAL", "Failed to restore special file %s: %v", fullPath, err)
 				return 0
 			}
 			remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
+			LOG_TRACE("DOWNLOAD_DONE", "Special %s %s restored", remoteEntry.Path, remoteEntry.FmtSpecial())
+
 		} else {
-			if remoteEntry.IsHardlinkRoot() {
+			if remoteEntry.IsHardLinkRoot() {
 				hardLinkTable[len(hardLinkTable)-1].willExist = true
-			} else if remoteEntry.IsHardlinkedFrom() {
+			} else if remoteEntry.IsHardLinkChild() {
-				i, err := remoteEntry.GetHardlinkId()
+				i, err := remoteEntry.GetHardLinkId()
 				if err != nil {
-					LOG_ERROR("RESTORE_HARDLINK", "Decode error for hardlinked entry %s, %v", remoteEntry.Path, err)
+					LOG_ERROR("RESTORE_HARDLINK", "Decode error for hard link entry %s: %v", remoteEntry.Path, err)
 					return 0
 				}
 				if !hardLinkTable[i].willExist {
@@ -992,7 +994,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 
 	for _, linkEntry := range hardLinks {
 
-		i, _ := linkEntry.GetHardlinkId()
+		i, _ := linkEntry.GetHardLinkId()
 		sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
 		fullPath := joinPath(top, linkEntry.Path)
 
@@ -1004,7 +1006,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 
 			if sourceStat == nil {
 				LOG_WERROR(allowFailures, "RESTORE_HARDLINK",
-					"Target %s for hardlink %s is missing", sourcePath, linkEntry.Path)
+					"Target %s for hard link %s is missing", sourcePath, linkEntry.Path)
 				continue
 			}
 			if !overwrite {
@@ -1015,11 +1017,11 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 			os.Remove(fullPath)
 		}
 
-		LOG_DEBUG("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath)
 		if err := MakeHardlink(sourcePath, fullPath); err != nil {
-			LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s", fullPath, sourcePath)
+			LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s: %v", fullPath, sourcePath, err)
 			return 0
 		}
+		LOG_TRACE("RESTORE_HARDLINK", "Hard linked %s to %s", linkEntry.Path, hardLinkTable[i].entry.Path)
 	}
 
 	if deleteMode && len(patterns) == 0 {
@@ -1197,8 +1199,7 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
 			entry.StartChunk -= delta
 			entry.EndChunk -= delta
 
-			if entry.IsHardlinkRoot() {
+			if entry.IsHardLinkRoot() {
-				LOG_DEBUG("SNAPSHOT_UPLOAD", "Hard link root %s %v %v", entry.Path, entry.StartChunk, entry.EndChunk)
 				hardLinkTable = append(hardLinkTable, hardLinkEntry{entry, entry.StartChunk})
 			}
 
@@ -1206,28 +1207,24 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
 			entry.StartChunk -= lastEndChunk
 			lastEndChunk = entry.EndChunk
 			entry.EndChunk = delta
-		} else if entry.IsHardlinkedFrom() && !entry.IsLink() {
+		} else if entry.IsHardLinkChild() {
-			i, err := entry.GetHardlinkId()
+			i, err := entry.GetHardLinkId()
 			if err != nil {
-				LOG_ERROR("SNAPSHOT_UPLOAD", "Decode error for hardlinked entry %s, %v", entry.Link, err)
+				LOG_ERROR("SNAPSHOT_UPLOAD", "Decode error for hard link entry %s: %v", entry.Link, err)
 				return err
 			}
 
 			targetEntry := hardLinkTable[i].entry
 			var startChunk, endChunk int
 
-			if targetEntry.Size > 0 {
+			if targetEntry.IsFile() && targetEntry.Size > 0 {
 				startChunk = hardLinkTable[i].startChunk - lastEndChunk
 				endChunk = targetEntry.EndChunk
+				lastEndChunk = hardLinkTable[i].startChunk + endChunk
 			}
 			entry = entry.HardLinkTo(targetEntry, startChunk, endChunk)
 
-			if targetEntry.Size > 0 {
+		} else if entry.IsHardLinkRoot() {
-				lastEndChunk = hardLinkTable[i].startChunk + endChunk
-			}
-
-			LOG_DEBUG("SNAPSHOT_UPLOAD", "Uploading cloned hardlink for %s to %s (%v %v)", entry.Path, targetEntry.Path, startChunk, endChunk)
-		} else if entry.IsHardlinkRoot() {
 			hardLinkTable = append(hardLinkTable, hardLinkEntry{entry, 0})
 		}
 

src/duplicacy_config.go

@@ -85,8 +85,8 @@ type Config struct {
 	FileKey []byte `json:"-"`
 
 	// for erasure coding
-	DataShards int `json:'data-shards'`
+	DataShards int `json:"data-shards"`
-	ParityShards int `json:'parity-shards'`
+	ParityShards int `json:"parity-shards"`
 
 	// for RSA encryption
 	rsaPrivateKey *rsa.PrivateKey

src/duplicacy_entry.go

@@ -25,8 +25,8 @@ import (
 )
 
 const (
-	entrySymHardLinkRootChunkMarker   = -72
+	entryHardLinkRootChunkMarker   = -9
-	entrySymHardLinkTargetChunkMarker = -73
+	entryHardLinkTargetChunkMarker = -10
 )
 
 // This is the hidden directory in the repository for storing various files.
@@ -126,6 +126,13 @@ func (entry *Entry) Copy() *Entry {
 }
 
 func (entry *Entry) HardLinkTo(target *Entry, startChunk int, endChunk int) *Entry {
+	endOffset := target.EndOffset
+
+	if !target.IsFile() {
+		startChunk = target.StartChunk
+		endChunk = entry.EndChunk
+		endOffset = entry.EndOffset
+	}
 	return &Entry{
 		Path: entry.Path,
 		Size: target.Size,
@@ -140,7 +147,7 @@ func (entry *Entry) HardLinkTo(target *Entry, startChunk int, endChunk int) *Ent
 		StartChunk:  startChunk,
 		StartOffset: target.StartOffset,
 		EndChunk:    endChunk,
-		EndOffset:   target.EndOffset,
+		EndOffset:   endOffset,
 
 		Attributes: target.Attributes,
 	}
@@ -516,34 +523,30 @@ func (entry *Entry) IsLink() bool {
 }
 
 func (entry *Entry) IsSpecial() bool {
-	return entry.Mode&uint32(os.ModeNamedPipe|os.ModeDevice|os.ModeCharDevice) != 0
+	return entry.Mode&uint32(os.ModeNamedPipe|os.ModeDevice|os.ModeCharDevice|os.ModeSocket) != 0
-}
-
-func (entry *Entry) IsFileOrSpecial() bool {
-	return entry.Mode&uint32(os.ModeDir|os.ModeSymlink|os.ModeIrregular) == 0
 }
 
 func (entry *Entry) IsComplete() bool {
 	return entry.Size >= 0
 }
 
-func (entry *Entry) IsHardlinkedFrom() bool {
+func (entry *Entry) IsHardLinkChild() bool {
-	return (entry.IsFileOrSpecial() && len(entry.Link) > 0 && entry.Link != "/") || (entry.IsLink() && entry.StartChunk == entrySymHardLinkTargetChunkMarker)
+	return (entry.IsFile() && len(entry.Link) > 0 && entry.Link != "/") || (!entry.IsDir() && entry.EndChunk == entryHardLinkTargetChunkMarker)
 }
 
-func (entry *Entry) IsHardlinkRoot() bool {
+func (entry *Entry) IsHardLinkRoot() bool {
-	return (entry.IsFileOrSpecial() && entry.Link == "/") || (entry.IsLink() && entry.StartChunk == entrySymHardLinkRootChunkMarker)
+	return (entry.IsFile() && entry.Link == "/") || (!entry.IsDir() && entry.EndChunk == entryHardLinkRootChunkMarker)
 }
 
-func (entry *Entry) GetHardlinkId() (int, error) {
+func (entry *Entry) GetHardLinkId() (int, error) {
-	if entry.IsLink() {
+	if entry.IsFile() {
-		if entry.StartChunk != entrySymHardLinkTargetChunkMarker {
-			return 0, errors.New("Symlink entry not marked as hardlinked")
-		}
-		return entry.StartOffset, nil
-	} else {
 		i, err := strconv.ParseUint(entry.Link, 16, 64)
 		return int(i), err
+	} else {
+		if entry.EndChunk != entryHardLinkTargetChunkMarker {
+			return 0, errors.New("Entry not marked as hard link child")
+		}
+		return entry.EndOffset, nil
 	}
 }
 
@@ -807,9 +810,6 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
 		if f.Name() == DUPLICACY_DIRECTORY {
 			continue
 		}
-		if f.Mode()&os.ModeSocket != 0 {
-			continue
-		}
 
 		entry := CreateEntryFromFileInfo(f, normalizedPath)
 		if len(patterns) > 0 && !MatchPath(entry.Path, patterns) {
@@ -823,21 +823,23 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
 				k := listEntryLinkKey{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}
 				if linkIndex, seen := listingState.linkTable[k]; seen {
 					if linkIndex == -1 {
-						LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute (hardlink)", entry.Path)
+						LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute (hard link)", entry.Path)
 						continue
 					}
 					entry.Size = 0
-					if entry.IsLink() {
+					if entry.IsFile() {
-						entry.StartChunk = entrySymHardLinkTargetChunkMarker
-						entry.StartOffset = linkIndex
-					} else {
 						entry.Link = strconv.FormatInt(int64(linkIndex), 16)
-					}
-				} else {
-					if entry.IsLink() {
-						entry.StartChunk = entrySymHardLinkRootChunkMarker
 					} else {
+						entry.EndChunk = entryHardLinkTargetChunkMarker
+						entry.EndOffset = linkIndex
+					}
+					listingChannel <- entry
+					continue
+				} else {
+					if entry.IsFile() {
 						entry.Link = "/"
+					} else {
+						entry.EndChunk = entryHardLinkRootChunkMarker
 					}
 					listingState.linkTable[k] = -1
 					linkKey = &k

src/duplicacy_entry_test.go

@@ -172,6 +172,8 @@ func TestEntryOrder(t *testing.T) {
 		}
 	}
 
+	listingState := NewListingState()
+
 	directories := make([]*Entry, 0, 4)
 	directories = append(directories, CreateEntry("", 0, 0, 0))
 
@@ -182,7 +184,7 @@ func TestEntryOrder(t *testing.T) {
 	for len(directories) > 0 {
 		directory := directories[len(directories)-1]
 		directories = directories[:len(directories)-1]
-		subdirectories, _, err := ListEntries(testDir, directory.Path, nil, "", false, entryChannel)
+		subdirectories, _, err := ListEntries(testDir, directory.Path, nil, "", false, listingState, entryChannel)
 		if err != nil {
 			t.Errorf("ListEntries(%s, %s) returned an error: %s", testDir, directory.Path, err)
 		}
@@ -288,6 +290,8 @@ func TestEntryExcludeByAttribute(t *testing.T) {
 
 	for _, excludeByAttribute := range [2]bool{true, false} {
 		t.Logf("testing excludeByAttribute: %t", excludeByAttribute)
+
+		listingState := NewListingState()
 		directories := make([]*Entry, 0, 4)
 		directories = append(directories, CreateEntry("", 0, 0, 0))
 
@@ -298,7 +302,7 @@ func TestEntryExcludeByAttribute(t *testing.T) {
 		for len(directories) > 0 {
 			directory := directories[len(directories)-1]
 			directories = directories[:len(directories)-1]
-			subdirectories, _, err := ListEntries(testDir, directory.Path, nil, "", excludeByAttribute, entryChannel)
+			subdirectories, _, err := ListEntries(testDir, directory.Path, nil, "", excludeByAttribute, listingState, entryChannel)
 			if err != nil {
 				t.Errorf("ListEntries(%s, %s) returned an error: %s", testDir, directory.Path, err)
 			}

src/duplicacy_snapshotmanager.go

@@ -2568,7 +2568,7 @@ func (manager *SnapshotManager) CheckSnapshot(snapshot *Snapshot) (err error) {
 		}
 
 		if entry.EndChunk < entry.StartChunk {
-			fmt.Errorf("The file %s starts at chunk %d and ends at chunk %d",
+			err = fmt.Errorf("The file %s starts at chunk %d and ends at chunk %d",
 				entry.Path, entry.StartChunk, entry.EndChunk)
 			return false
 		}

src/duplicacy_storage.go

@@ -756,6 +756,8 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the Storj storage at %s: %v", storageURL, err)
 			return nil
 		}
+		SavePassword(preference, "storj_key", apiKey)
+		SavePassword(preference, "storj_passphrase", passphrase)
 		return storjStorage
 	} else if matched[1] == "smb" {
 		server := matched[3]

src/duplicacy_utils_bsd_common.go

@@ -92,3 +92,18 @@ func (entry *Entry) RestoreEarlyDirFlags(path string) error {
 func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
 	return nil
 }
+
+func (entry *Entry) RestoreSpecial(fullPath string) error {
+	mode := entry.Mode & uint32(fileModeMask)
+
+	if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
+		mode |= syscall.S_IFIFO
+	} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
+		mode |= syscall.S_IFCHR
+	} else if entry.Mode&uint32(os.ModeDevice) != 0 {
+		mode |= syscall.S_IFBLK
+	} else {
+		return nil
+	}
+	return syscall.Mknod(fullPath, mode, int(entry.GetRdev()))
+}

src/duplicacy_utils_linux.go

@@ -219,6 +219,23 @@ func (entry *Entry) restoreLateFileFlags(f *os.File) error {
 	return nil
 }
 
+func (entry *Entry) RestoreSpecial(fullPath string) error {
+	mode := entry.Mode & uint32(fileModeMask)
+
+	if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
+		mode |= syscall.S_IFIFO
+	} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
+		mode |= syscall.S_IFCHR
+	} else if entry.Mode&uint32(os.ModeDevice) != 0 {
+		mode |= syscall.S_IFBLK
+	} else if entry.Mode&uint32(os.ModeSocket) != 0 {
+		mode |= syscall.S_IFSOCK
+	} else {
+		return nil
+	}
+	return syscall.Mknod(fullPath, mode, int(entry.GetRdev()))
+}
+
 func excludedByAttribute(attributes map[string][]byte) bool {
 	_, ok := attributes["user.duplicacy_exclude"]
 	return ok

src/duplicacy_utils_others.go

@@ -8,6 +8,7 @@
 package duplicacy
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"syscall"
@@ -65,21 +66,6 @@ func (entry *Entry) GetRdev() uint64 {
 	return uint64(entry.StartChunk) | uint64(entry.StartOffset)<<32
 }
 
-func (entry *Entry) RestoreSpecial(fullPath string) error {
-	mode := entry.Mode & uint32(fileModeMask)
-
-	if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
-		mode |= syscall.S_IFIFO
-	} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
-		mode |= syscall.S_IFCHR
-	} else if entry.Mode&uint32(os.ModeDevice) != 0 {
-		mode |= syscall.S_IFBLK
-	} else {
-		return nil
-	}
-	return syscall.Mknod(fullPath, mode, int(entry.GetRdev()))
-}
-
 func (entry *Entry) IsSameSpecial(fileInfo os.FileInfo) bool {
 	stat := fileInfo.Sys().(*syscall.Stat_t)
 	if stat == nil {
@@ -88,6 +74,26 @@ func (entry *Entry) IsSameSpecial(fileInfo os.FileInfo) bool {
 	return (uint32(fileInfo.Mode()) == entry.Mode) && (uint64(stat.Rdev) == entry.GetRdev())
 }
 
+func (entry *Entry) FmtSpecial() string {
+	var c string
+	mode := entry.Mode & uint32(os.ModeType)
+
+	if mode&uint32(os.ModeNamedPipe) != 0 {
+		c = "p"
+	} else if mode&uint32(os.ModeCharDevice) != 0 {
+		c = "c"
+	} else if mode&uint32(os.ModeDevice) != 0 {
+		c = "b"
+	} else if mode&uint32(os.ModeSocket) != 0 {
+		c = "s"
+	} else {
+		return ""
+	}
+
+	rdev := entry.GetRdev()
+	return fmt.Sprintf("%s (%d, %d)", c, unix.Major(rdev), unix.Minor(rdev))
+}
+
 func MakeHardlink(source string, target string) error {
 	return unix.Linkat(unix.AT_FDCWD, source, unix.AT_FDCWD, target, 0)
 }

src/duplicacy_utils_windows.go

@@ -125,6 +125,10 @@ func (entry *Entry) RestoreSpecial(fullPath string) error {
 	return nil
 }
 
+func (entry *Entry) FmtSpecial() string {
+	return ""
+}
+
 func MakeHardlink(source string, target string) error {
 	return os.Link(source, target)
 }