jkl1337/duplicacy
1
0
Fork 0
mirror of https://github.com/jkl1337/duplicacy.git synced 2026-01-02 11:44:45 -06:00
Code Issues Packages Projects Releases Wiki Activity

Support for hardlinks to symlinks

Browse Source 
This is a specialized use-case, but it is indeed possible to do this
and can be used if xattrs are attached to the symlink.
This commit is contained in:
John K. Luebs
2023-10-03 23:46:00 -05:00
committed by John K. Luebs
parent 5b40bf3d93
commit a1a3f3d4cb
5 changed files with 165 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
src/duplicacy_backupmanager.go
View File

@@ -705,7 +705,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
type hardLinkEntry struct {
entry *Entry
willDownload bool
willExist bool
}
var hardLinkTable []hardLinkEntry
var hardLinks []*Entry
@@ -752,12 +752,16 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
}
fullPath := joinPath(top, remoteEntry.Path)
if remoteEntry.IsLink() {
if stat, _ := os.Lstat(fullPath); stat != nil {
if stat.Mode()&os.ModeSymlink != 0 {
isRegular, link, err := Readlink(fullPath)
if err == nil && link == remoteEntry.Link && !isRegular {
remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
if remoteEntry.IsHardlinkRoot() {
hardLinkTable[len(hardLinkTable)-1].willExist = true
}
continue
}
}
@@ -771,11 +775,33 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
os.Remove(fullPath)
}
if remoteEntry.IsHardlinkRoot() {
hardLinkTable[len(hardLinkTable)-1].willExist = true
} else if remoteEntry.IsHardlinkedFrom() {
i, err := remoteEntry.GetHardlinkId()
if err != nil {
LOG_ERROR("RESTORE_HARDLINK", "Decode error for hardlinked entry %s, %v", remoteEntry.Path, err)
return 0
}
if !hardLinkTable[i].willExist {
hardLinkTable[i] = hardLinkEntry{remoteEntry, true}
} else {
sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
LOG_INFO("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath)
if err := MakeHardlink(sourcePath, fullPath); err != nil {
LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s %v", fullPath, sourcePath, err)
return 0
}
continue
}
}
if err := os.Symlink(remoteEntry.Link, fullPath); err != nil {
LOG_ERROR("RESTORE_SYMLINK", "Can't create symlink %s: %v", remoteEntry.Path, err)
return 0
}
remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
LOG_TRACE("DOWNLOAD_DONE", "Symlink %s updated", remoteEntry.Path)
} else if remoteEntry.IsDir() {
@@ -814,20 +840,21 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
} else {
if remoteEntry.IsHardlinkRoot() {
hardLinkTable[len(hardLinkTable)-1] = hardLinkEntry{remoteEntry, true}
hardLinkTable[len(hardLinkTable)-1].willExist = true
} else if remoteEntry.IsHardlinkedFrom() {
i, err := strconv.ParseUint(remoteEntry.Link, 16, 64)
i, err := remoteEntry.GetHardlinkId()
if err != nil {
LOG_ERROR("RESTORE_HARDLINK", "Decode error in hardlink entry, expected hex int, got %s", remoteEntry.Link)
LOG_ERROR("RESTORE_HARDLINK", "Decode error for hardlinked entry %s, %v", remoteEntry.Path, err)
return 0
}
if !hardLinkTable[i].willDownload {
if !hardLinkTable[i].willExist {
hardLinkTable[i] = hardLinkEntry{remoteEntry, true}
} else {
hardLinks = append(hardLinks, remoteEntry)
continue
}
}
// We can't download files here since fileEntries needs to be sorted
fileEntries = append(fileEntries, remoteEntry)
totalFileSize += remoteEntry.Size
@@ -948,11 +975,32 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
}
for _, linkEntry := range hardLinks {
i, _ := strconv.ParseUint(linkEntry.Link, 16, 64)
i, _ := linkEntry.GetHardlinkId()
sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
fullPath := joinPath(top, linkEntry.Path)
LOG_INFO("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath)
if err := os.Link(sourcePath, fullPath); err != nil {
if stat, _ := os.Lstat(fullPath); stat != nil {
sourceStat, _ := os.Lstat(sourcePath)
if os.SameFile(stat, sourceStat) {
continue
}
if sourceStat == nil {
LOG_WERROR(allowFailures, "RESTORE_HARDLINK",
"Target %s for hardlink %s is missing", sourcePath, linkEntry.Path)
continue
}
if !overwrite {
LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
"File %s already exists. Please specify the -overwrite option to overwrite", linkEntry.Path)
continue
}
os.Remove(fullPath)
}
LOG_DEBUG("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath)
if err := MakeHardlink(sourcePath, fullPath); err != nil {
LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s", fullPath, sourcePath)
return 0
}
@@ -1142,10 +1190,10 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
entry.StartChunk -= lastEndChunk
lastEndChunk = entry.EndChunk
entry.EndChunk = delta
} else if entry.IsHardlinkedFrom() {
i, err := strconv.ParseUint(entry.Link, 16, 64)
} else if entry.IsHardlinkedFrom() && !entry.IsLink() {
i, err := entry.GetHardlinkId()
if err != nil {
LOG_ERROR("SNAPSHOT_UPLOAD", "Decode error in hardlink entry, expected hex int, got %s", entry.Link)
LOG_ERROR("SNAPSHOT_UPLOAD", "Decode error for hardlinked entry %s, %v", entry.Link, err)
return err
}

76
src/duplicacy_entry.go
View File

@@ -8,6 +8,7 @@ import (
"crypto/sha256"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
@@ -23,6 +24,11 @@ import (
"github.com/vmihailenco/msgpack"
)
const (
entrySymHardLinkRootChunkMarker = -72
entrySymHardLinkTargetChunkMarker = -73
)
// This is the hidden directory in the repository for storing various files.
var DUPLICACY_DIRECTORY = ".duplicacy"
var DUPLICACY_FILE = ".duplicacy"
@@ -518,11 +524,23 @@ func (entry *Entry) IsComplete() bool {
}
func (entry *Entry) IsHardlinkedFrom() bool {
return entry.IsFile() && len(entry.Link) > 0 && entry.Link != "/"
return (entry.IsFile() && len(entry.Link) > 0 && entry.Link != "/") || (entry.IsLink() && entry.StartChunk == entrySymHardLinkTargetChunkMarker)
}
func (entry *Entry) IsHardlinkRoot() bool {
return entry.IsFile() && entry.Link == "/"
return (entry.IsFile() && entry.Link == "/") || (entry.IsLink() && entry.StartChunk == entrySymHardLinkRootChunkMarker)
}
func (entry *Entry) GetHardlinkId() (int, error) {
if entry.IsLink() {
if entry.StartChunk != entrySymHardLinkTargetChunkMarker {
return 0, errors.New("Symlink entry not marked as hardlinked")
}
return entry.StartOffset, nil
} else {
i, err := strconv.ParseUint(entry.Link, 16, 64)
return int(i), err
}
}
func (entry *Entry) GetPermissions() os.FileMode {
@@ -789,6 +807,42 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
if len(patterns) > 0 && !MatchPath(entry.Path, patterns) {
continue
}
if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
LOG_WARN("LIST_SKIP", "Skipped non-regular file %s", entry.Path)
skippedFiles = append(skippedFiles, entry.Path)
continue
}
var linkKey *listEntryLinkKey
if runtime.GOOS != "windows" && !entry.IsDir() {
if stat := f.Sys().(*syscall.Stat_t); stat != nil && stat.Nlink > 1 {
k := listEntryLinkKey{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}
if linkIndex, seen := listingState.linkTable[k]; seen {
if linkIndex == -1 {
LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute (hardlink)", entry.Path)
continue
}
entry.Size = 0
if entry.IsLink() {
entry.StartChunk = entrySymHardLinkTargetChunkMarker
entry.StartOffset = linkIndex
} else {
entry.Link = strconv.FormatInt(int64(linkIndex), 16)
}
} else {
if entry.IsLink() {
entry.StartChunk = entrySymHardLinkRootChunkMarker
} else {
entry.Link = "/"
}
listingState.linkTable[k] = -1
linkKey = &k
}
}
}
if entry.IsLink() {
isRegular := false
isRegular, entry.Link, err = Readlink(joinPath(top, entry.Path))
@@ -830,24 +884,6 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
}
}
var linkKey *listEntryLinkKey
if stat, ok := f.Sys().(*syscall.Stat_t); entry.IsFile() && ok && stat != nil && stat.Nlink > 1 {
k := listEntryLinkKey{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}
if linkIndex, seen := listingState.linkTable[k]; seen {
if linkIndex == -1 {
LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute (hardlink)", entry.Path)
continue
}
entry.Size = 0
entry.Link = strconv.FormatInt(int64(linkIndex), 16)
} else {
entry.Link = "/"
listingState.linkTable[k] = -1
linkKey = &k
}
}
entry.ReadAttributes(top)
if excludeByAttribute && entry.Attributes != nil && excludedByAttribute(*entry.Attributes) {

35
src/duplicacy_utils.go
View File

@@ -7,14 +7,15 @@ package duplicacy
import (
"bufio"
"crypto/sha256"
"encoding/json"
"fmt"
"io"
"os"
"regexp"
"runtime"
"strconv"
"strings"
"time"
"runtime"
"github.com/gilbertchen/gopass"
"golang.org/x/crypto/pbkdf2"
@@ -275,7 +276,6 @@ func SavePassword(preference Preference, passwordType string, password string) {
// The following code was modified from the online article 'Matching Wildcards: An Algorithm', by Kirk J. Krauss,
// Dr. Dobb's, August 26, 2008. However, the version in the article doesn't handle cases like matching 'abcccd'
// against '*ccd', and the version here fixed that issue.
//
func matchPattern(text string, pattern string) bool {
textLength := len(text)
@@ -474,3 +474,34 @@ func PrintMemoryUsage() {
time.Sleep(time.Second)
}
}
func (entry *Entry) dump() map[string]interface{} {
object := make(map[string]interface{})
object["path"] = entry.Path
object["size"] = entry.Size
object["time"] = entry.Time
object["mode"] = entry.Mode
object["hash"] = entry.Hash
object["link"] = entry.Link
object["content"] = fmt.Sprintf("%d:%d:%d:%d",
entry.StartChunk, entry.StartOffset, entry.EndChunk, entry.EndOffset)
if entry.UID != -1 && entry.GID != -1 {
object["uid"] = entry.UID
object["gid"] = entry.GID
}
if entry.Attributes != nil && len(*entry.Attributes) > 0 {
object["attributes"] = entry.Attributes
}
return object
}
func (entry *Entry) dumpString() string {
data, _ := json.Marshal(entry.dump())
return string(data)
}

14
src/duplicacy_utils_others.go
View File

@@ -11,6 +11,8 @@ import (
"os"
"path"
"syscall"
"golang.org/x/sys/unix"
)
func Readlink(path string) (isRegular bool, s string, err error) {
@@ -45,7 +47,7 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
}
func (entry *Entry) ReadSpecial(fileInfo os.FileInfo) bool {
if fileInfo.Mode() & (os.ModeDevice | os.ModeCharDevice) == 0 {
if fileInfo.Mode()&(os.ModeDevice|os.ModeCharDevice) == 0 {
return true
}
stat, ok := fileInfo.Sys().(*syscall.Stat_t)
@@ -62,16 +64,20 @@ func (entry *Entry) ReadSpecial(fileInfo os.FileInfo) bool {
func (entry *Entry) RestoreSpecial(fullPath string) error {
mode := entry.Mode & uint32(fileModeMask)
if entry.Mode & uint32(os.ModeNamedPipe) != 0 {
if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
mode |= syscall.S_IFIFO
} else if entry.Mode & uint32(os.ModeCharDevice) != 0 {
} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
mode |= syscall.S_IFCHR
} else if entry.Mode & uint32(os.ModeDevice) != 0 {
mode |= syscall.S_IFBLK
} else {
return nil
}
return syscall.Mknod(fullPath, mode, int(uint64(entry.StartChunk) | uint64(entry.StartOffset) << 32))
return syscall.Mknod(fullPath, mode, int(uint64(entry.StartChunk)|uint64(entry.StartOffset)<<32))
}
func MakeHardlink(source string, target string) error {
return unix.Linkat(unix.AT_FDCWD, source, unix.AT_FDCWD, target, 0)
}
func joinPath(components ...string) string {

4
src/duplicacy_utils_windows.go
View File

@@ -125,6 +125,10 @@ func (entry *Entry) RestoreSpecial(fullPath string) error {
return nil
}
func MakeHardlink(source string, target string) error {
return os.Link(source, target)
}
func joinPath(components ...string) string {
combinedPath := `\\?\` + filepath.Join(components...)