diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index e44550d..cde8957 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -705,7 +705,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu type hardLinkEntry struct { entry *Entry - willDownload bool + willExist bool } var hardLinkTable []hardLinkEntry var hardLinks []*Entry @@ -752,12 +752,16 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu } fullPath := joinPath(top, remoteEntry.Path) + if remoteEntry.IsLink() { if stat, _ := os.Lstat(fullPath); stat != nil { if stat.Mode()&os.ModeSymlink != 0 { isRegular, link, err := Readlink(fullPath) if err == nil && link == remoteEntry.Link && !isRegular { remoteEntry.RestoreMetadata(fullPath, nil, setOwner) + if remoteEntry.IsHardlinkRoot() { + hardLinkTable[len(hardLinkTable)-1].willExist = true + } continue } } @@ -771,11 +775,33 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu os.Remove(fullPath) } + if remoteEntry.IsHardlinkRoot() { + hardLinkTable[len(hardLinkTable)-1].willExist = true + } else if remoteEntry.IsHardlinkedFrom() { + i, err := remoteEntry.GetHardlinkId() + if err != nil { + LOG_ERROR("RESTORE_HARDLINK", "Decode error for hardlinked entry %s, %v", remoteEntry.Path, err) + return 0 + } + if !hardLinkTable[i].willExist { + hardLinkTable[i] = hardLinkEntry{remoteEntry, true} + } else { + sourcePath := joinPath(top, hardLinkTable[i].entry.Path) + LOG_INFO("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath) + if err := MakeHardlink(sourcePath, fullPath); err != nil { + LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s %v", fullPath, sourcePath, err) + return 0 + } + continue + } + } + if err := os.Symlink(remoteEntry.Link, fullPath); err != nil { LOG_ERROR("RESTORE_SYMLINK", "Can't create symlink %s: %v", remoteEntry.Path, err) return 0 } remoteEntry.RestoreMetadata(fullPath, nil, setOwner) + LOG_TRACE("DOWNLOAD_DONE", "Symlink %s updated", remoteEntry.Path) } else if remoteEntry.IsDir() { @@ -814,20 +840,21 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu remoteEntry.RestoreMetadata(fullPath, nil, setOwner) } else { if remoteEntry.IsHardlinkRoot() { - hardLinkTable[len(hardLinkTable)-1] = hardLinkEntry{remoteEntry, true} + hardLinkTable[len(hardLinkTable)-1].willExist = true } else if remoteEntry.IsHardlinkedFrom() { - i, err := strconv.ParseUint(remoteEntry.Link, 16, 64) + i, err := remoteEntry.GetHardlinkId() if err != nil { - LOG_ERROR("RESTORE_HARDLINK", "Decode error in hardlink entry, expected hex int, got %s", remoteEntry.Link) + LOG_ERROR("RESTORE_HARDLINK", "Decode error for hardlinked entry %s, %v", remoteEntry.Path, err) return 0 } - if !hardLinkTable[i].willDownload { + if !hardLinkTable[i].willExist { hardLinkTable[i] = hardLinkEntry{remoteEntry, true} } else { hardLinks = append(hardLinks, remoteEntry) continue } } + // We can't download files here since fileEntries needs to be sorted fileEntries = append(fileEntries, remoteEntry) totalFileSize += remoteEntry.Size @@ -948,11 +975,32 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu } for _, linkEntry := range hardLinks { - i, _ := strconv.ParseUint(linkEntry.Link, 16, 64) + + i, _ := linkEntry.GetHardlinkId() sourcePath := joinPath(top, hardLinkTable[i].entry.Path) fullPath := joinPath(top, linkEntry.Path) - LOG_INFO("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath) - if err := os.Link(sourcePath, fullPath); err != nil { + + if stat, _ := os.Lstat(fullPath); stat != nil { + sourceStat, _ := os.Lstat(sourcePath) + if os.SameFile(stat, sourceStat) { + continue + } + + if sourceStat == nil { + LOG_WERROR(allowFailures, "RESTORE_HARDLINK", + "Target %s for hardlink %s is missing", sourcePath, linkEntry.Path) + continue + } + if !overwrite { + LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE", + "File %s already exists. Please specify the -overwrite option to overwrite", linkEntry.Path) + continue + } + os.Remove(fullPath) + } + + LOG_DEBUG("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath) + if err := MakeHardlink(sourcePath, fullPath); err != nil { LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s", fullPath, sourcePath) return 0 } @@ -1142,10 +1190,10 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s entry.StartChunk -= lastEndChunk lastEndChunk = entry.EndChunk entry.EndChunk = delta - } else if entry.IsHardlinkedFrom() { - i, err := strconv.ParseUint(entry.Link, 16, 64) + } else if entry.IsHardlinkedFrom() && !entry.IsLink() { + i, err := entry.GetHardlinkId() if err != nil { - LOG_ERROR("SNAPSHOT_UPLOAD", "Decode error in hardlink entry, expected hex int, got %s", entry.Link) + LOG_ERROR("SNAPSHOT_UPLOAD", "Decode error for hardlinked entry %s, %v", entry.Link, err) return err } diff --git a/src/duplicacy_entry.go b/src/duplicacy_entry.go index 6e00ba1..095bf43 100644 --- a/src/duplicacy_entry.go +++ b/src/duplicacy_entry.go @@ -8,6 +8,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/json" + "errors" "fmt" "io/ioutil" "os" @@ -23,6 +24,11 @@ import ( "github.com/vmihailenco/msgpack" ) +const ( + entrySymHardLinkRootChunkMarker = -72 + entrySymHardLinkTargetChunkMarker = -73 +) + // This is the hidden directory in the repository for storing various files. var DUPLICACY_DIRECTORY = ".duplicacy" var DUPLICACY_FILE = ".duplicacy" @@ -518,11 +524,23 @@ func (entry *Entry) IsComplete() bool { } func (entry *Entry) IsHardlinkedFrom() bool { - return entry.IsFile() && len(entry.Link) > 0 && entry.Link != "/" + return (entry.IsFile() && len(entry.Link) > 0 && entry.Link != "/") || (entry.IsLink() && entry.StartChunk == entrySymHardLinkTargetChunkMarker) } func (entry *Entry) IsHardlinkRoot() bool { - return entry.IsFile() && entry.Link == "/" + return (entry.IsFile() && entry.Link == "/") || (entry.IsLink() && entry.StartChunk == entrySymHardLinkRootChunkMarker) +} + +func (entry *Entry) GetHardlinkId() (int, error) { + if entry.IsLink() { + if entry.StartChunk != entrySymHardLinkTargetChunkMarker { + return 0, errors.New("Symlink entry not marked as hardlinked") + } + return entry.StartOffset, nil + } else { + i, err := strconv.ParseUint(entry.Link, 16, 64) + return int(i), err + } } func (entry *Entry) GetPermissions() os.FileMode { @@ -789,6 +807,42 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string if len(patterns) > 0 && !MatchPath(entry.Path, patterns) { continue } + + if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 { + LOG_WARN("LIST_SKIP", "Skipped non-regular file %s", entry.Path) + skippedFiles = append(skippedFiles, entry.Path) + continue + } + + var linkKey *listEntryLinkKey + + if runtime.GOOS != "windows" && !entry.IsDir() { + if stat := f.Sys().(*syscall.Stat_t); stat != nil && stat.Nlink > 1 { + k := listEntryLinkKey{dev: uint64(stat.Dev), ino: uint64(stat.Ino)} + if linkIndex, seen := listingState.linkTable[k]; seen { + if linkIndex == -1 { + LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute (hardlink)", entry.Path) + continue + } + entry.Size = 0 + if entry.IsLink() { + entry.StartChunk = entrySymHardLinkTargetChunkMarker + entry.StartOffset = linkIndex + } else { + entry.Link = strconv.FormatInt(int64(linkIndex), 16) + } + } else { + if entry.IsLink() { + entry.StartChunk = entrySymHardLinkRootChunkMarker + } else { + entry.Link = "/" + } + listingState.linkTable[k] = -1 + linkKey = &k + } + } + } + if entry.IsLink() { isRegular := false isRegular, entry.Link, err = Readlink(joinPath(top, entry.Path)) @@ -830,24 +884,6 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string } } - var linkKey *listEntryLinkKey - - if stat, ok := f.Sys().(*syscall.Stat_t); entry.IsFile() && ok && stat != nil && stat.Nlink > 1 { - k := listEntryLinkKey{dev: uint64(stat.Dev), ino: uint64(stat.Ino)} - if linkIndex, seen := listingState.linkTable[k]; seen { - if linkIndex == -1 { - LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute (hardlink)", entry.Path) - continue - } - entry.Size = 0 - entry.Link = strconv.FormatInt(int64(linkIndex), 16) - } else { - entry.Link = "/" - listingState.linkTable[k] = -1 - linkKey = &k - } - } - entry.ReadAttributes(top) if excludeByAttribute && entry.Attributes != nil && excludedByAttribute(*entry.Attributes) { diff --git a/src/duplicacy_utils.go b/src/duplicacy_utils.go index e1f2b88..86b22b3 100644 --- a/src/duplicacy_utils.go +++ b/src/duplicacy_utils.go @@ -7,14 +7,15 @@ package duplicacy import ( "bufio" "crypto/sha256" + "encoding/json" "fmt" "io" "os" "regexp" + "runtime" "strconv" "strings" "time" - "runtime" "github.com/gilbertchen/gopass" "golang.org/x/crypto/pbkdf2" @@ -56,7 +57,7 @@ func IsEmptyFilter(pattern string) bool { } func IsUnspecifiedFilter(pattern string) bool { - if pattern[0] != '+' && pattern[0] != '-' && !strings.HasPrefix(pattern, "i:") && !strings.HasPrefix(pattern, "e:") { + if pattern[0] != '+' && pattern[0] != '-' && !strings.HasPrefix(pattern, "i:") && !strings.HasPrefix(pattern, "e:") { return true } else { return false @@ -275,7 +276,6 @@ func SavePassword(preference Preference, passwordType string, password string) { // The following code was modified from the online article 'Matching Wildcards: An Algorithm', by Kirk J. Krauss, // Dr. Dobb's, August 26, 2008. However, the version in the article doesn't handle cases like matching 'abcccd' // against '*ccd', and the version here fixed that issue. -// func matchPattern(text string, pattern string) bool { textLength := len(text) @@ -469,8 +469,39 @@ func PrintMemoryUsage() { runtime.ReadMemStats(&m) LOG_INFO("MEMORY_STATS", "Currently allocated: %s, total allocated: %s, system memory: %s, number of GCs: %d", - PrettySize(int64(m.Alloc)), PrettySize(int64(m.TotalAlloc)), PrettySize(int64(m.Sys)), m.NumGC) + PrettySize(int64(m.Alloc)), PrettySize(int64(m.TotalAlloc)), PrettySize(int64(m.Sys)), m.NumGC) time.Sleep(time.Second) } -} \ No newline at end of file +} + +func (entry *Entry) dump() map[string]interface{} { + + object := make(map[string]interface{}) + + object["path"] = entry.Path + object["size"] = entry.Size + object["time"] = entry.Time + object["mode"] = entry.Mode + object["hash"] = entry.Hash + object["link"] = entry.Link + + object["content"] = fmt.Sprintf("%d:%d:%d:%d", + entry.StartChunk, entry.StartOffset, entry.EndChunk, entry.EndOffset) + + if entry.UID != -1 && entry.GID != -1 { + object["uid"] = entry.UID + object["gid"] = entry.GID + } + + if entry.Attributes != nil && len(*entry.Attributes) > 0 { + object["attributes"] = entry.Attributes + } + + return object +} + +func (entry *Entry) dumpString() string { + data, _ := json.Marshal(entry.dump()) + return string(data) +} diff --git a/src/duplicacy_utils_others.go b/src/duplicacy_utils_others.go index 4feae07..9767516 100644 --- a/src/duplicacy_utils_others.go +++ b/src/duplicacy_utils_others.go @@ -11,6 +11,8 @@ import ( "os" "path" "syscall" + + "golang.org/x/sys/unix" ) func Readlink(path string) (isRegular bool, s string, err error) { @@ -45,7 +47,7 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool { } func (entry *Entry) ReadSpecial(fileInfo os.FileInfo) bool { - if fileInfo.Mode() & (os.ModeDevice | os.ModeCharDevice) == 0 { + if fileInfo.Mode()&(os.ModeDevice|os.ModeCharDevice) == 0 { return true } stat, ok := fileInfo.Sys().(*syscall.Stat_t) @@ -62,16 +64,20 @@ func (entry *Entry) ReadSpecial(fileInfo os.FileInfo) bool { func (entry *Entry) RestoreSpecial(fullPath string) error { mode := entry.Mode & uint32(fileModeMask) - if entry.Mode & uint32(os.ModeNamedPipe) != 0 { + if entry.Mode&uint32(os.ModeNamedPipe) != 0 { mode |= syscall.S_IFIFO - } else if entry.Mode & uint32(os.ModeCharDevice) != 0 { + } else if entry.Mode&uint32(os.ModeCharDevice) != 0 { mode |= syscall.S_IFCHR } else if entry.Mode & uint32(os.ModeDevice) != 0 { mode |= syscall.S_IFBLK } else { return nil } - return syscall.Mknod(fullPath, mode, int(uint64(entry.StartChunk) | uint64(entry.StartOffset) << 32)) + return syscall.Mknod(fullPath, mode, int(uint64(entry.StartChunk)|uint64(entry.StartOffset)<<32)) +} + +func MakeHardlink(source string, target string) error { + return unix.Linkat(unix.AT_FDCWD, source, unix.AT_FDCWD, target, 0) } func joinPath(components ...string) string { diff --git a/src/duplicacy_utils_windows.go b/src/duplicacy_utils_windows.go index 3846ca6..52df9f4 100644 --- a/src/duplicacy_utils_windows.go +++ b/src/duplicacy_utils_windows.go @@ -125,6 +125,10 @@ func (entry *Entry) RestoreSpecial(fullPath string) error { return nil } +func MakeHardlink(source string, target string) error { + return os.Link(source, target) +} + func joinPath(components ...string) string { combinedPath := `\\?\` + filepath.Join(components...)