Initial implementation of file/inode flags (Linux, BSD, darwin)

Basic support for BSD and Darwin style chflags (stat flags). Applies
these flags at the end of file restore.
Supports linux style ioctl_iflags(2) in a 2 step process. Flags that
need to be applied prior to writes such as compress and especially no-COW
are applied immediately upon file open.

The flags format is backwards compatible. An attribute starting with a
null byte is used to store flags in the entry attributes table. With
an old version of duplicacy the restore of this attribute should silently
fail (effectively be ignored).

Fixes xattr restore to use O_NOFOLLOW so attributes are applied to symlink.

TODO: Tests, possible option to switch off mutable/append prior to
restore of existing file similar to rsync. Does not apply attributes
or flags to the top most directory.

src/duplicacy_backupmanager.go

@@ -703,19 +703,8 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 	var localEntry *Entry
 	localListingOK := true
 
-	type hardLinkEntry struct {
-		entry *Entry
-		willDownload bool
-	}
-	var hardLinkTable []hardLinkEntry
-	var hardLinks []*Entry
-
 	for remoteEntry := range remoteListingChannel {
 
-		if remoteEntry.IsHardlinkRoot() {
-			hardLinkTable = append(hardLinkTable, hardLinkEntry{remoteEntry, false})
-		}
-
 		if len(patterns) > 0 && !MatchPath(remoteEntry.Path, patterns) {
 			continue
 		}
@@ -753,7 +742,8 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 
 		fullPath := joinPath(top, remoteEntry.Path)
 		if remoteEntry.IsLink() {
-			if stat, _ := os.Lstat(fullPath); stat != nil {
+			stat, err := os.Lstat(fullPath)
+			if stat != nil {
 				if stat.Mode()&os.ModeSymlink != 0 {
 					isRegular, link, err := Readlink(fullPath)
 					if err == nil && link == remoteEntry.Link && !isRegular {
@@ -762,16 +752,11 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 					}
 				}
 
-				if !overwrite {
-					LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
-						"File %s already exists.  Please specify the -overwrite option to overwrite", remoteEntry.Path)
-					continue
-				}
-
 				os.Remove(fullPath)
 			}
 
-			if err := os.Symlink(remoteEntry.Link, fullPath); err != nil {
+			err = os.Symlink(remoteEntry.Link, fullPath)
+			if err != nil {
 				LOG_ERROR("RESTORE_SYMLINK", "Can't create symlink %s: %v", remoteEntry.Path, err)
 				return 0
 			}
@@ -797,37 +782,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 			}
 			remoteEntry.RestoreEarlyDirFlags(fullPath)
 			directoryEntries = append(directoryEntries, remoteEntry)
-		} else if remoteEntry.IsSpecial() {
-			if stat, _ := os.Lstat(fullPath); stat != nil {
-				if !overwrite {
-					LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
-						"File %s already exists.  Please specify the -overwrite option to overwrite", remoteEntry.Path)
-					continue
-				}
-				os.Remove(fullPath)
-			}
-
-			if err := remoteEntry.RestoreSpecial(fullPath); err != nil {
-				LOG_ERROR("RESTORE_SPECIAL", "Unable to restore special file %s: %v", remoteEntry.Path, err)
-				return 0
-			}
-			remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
 		} else {
-			if remoteEntry.IsHardlinkRoot() {
-				hardLinkTable[len(hardLinkTable)-1] = hardLinkEntry{remoteEntry, true}
-			} else if remoteEntry.IsHardlinkedFrom() {
-				i, err := strconv.ParseUint(remoteEntry.Link, 16, 64)
-				if err !=  nil {
-					LOG_ERROR("RESTORE_HARDLINK", "Decode error in hardlink entry, expected hex int, got %s", remoteEntry.Link)
-					return 0
-				}
-				if !hardLinkTable[i].willDownload {
-					hardLinkTable[i] = hardLinkEntry{remoteEntry, true}
-				} else {
-					hardLinks = append(hardLinks, remoteEntry)
-					continue
-				}
-			}
 			// We can't download files here since fileEntries needs to be sorted
 			fileEntries = append(fileEntries, remoteEntry)
 			totalFileSize += remoteEntry.Size
@@ -947,17 +902,6 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		file.RestoreMetadata(fullPath, nil, setOwner)
 	}
 
-	for _, linkEntry := range hardLinks {
-		i, _ := strconv.ParseUint(linkEntry.Link, 16, 64)
-		sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
-		fullPath := joinPath(top, linkEntry.Path)
-		LOG_INFO("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath)
-		if err := os.Link(sourcePath, fullPath); err != nil {
-			LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s", fullPath, sourcePath)
-			return 0
-		}
-	}
-
 	if deleteMode && len(patterns) == 0 {
 		// Reverse the order to make sure directories are empty before being deleted
 		for i := range extraFiles {
@@ -1109,13 +1053,8 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
 
 	lastEndChunk := 0
 
-	type hardLinkEntry struct {
-		entry *Entry
-		startChunk int
-	}
-	var hardLinkTable []hardLinkEntry
-
 	uploadEntryInfoFunc := func(entry *Entry) error {
+
 		if entry.IsFile() && entry.Size > 0 {
 			delta := entry.StartChunk - len(chunkHashes) + 1
 			if entry.StartChunk != lastChunk {
@@ -1133,38 +1072,10 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
 			entry.StartChunk -= delta
 			entry.EndChunk -= delta
 
-			if entry.IsHardlinkRoot() {
-				LOG_DEBUG("SNAPSHOT_UPLOAD", "Hard link root %s %v %v", entry.Path, entry.StartChunk, entry.EndChunk)
-				hardLinkTable = append(hardLinkTable, hardLinkEntry{entry, entry.StartChunk})
-			}
-
 			delta = entry.EndChunk - entry.StartChunk
 			entry.StartChunk -= lastEndChunk
 			lastEndChunk = entry.EndChunk
 			entry.EndChunk = delta
-		} else if entry.IsHardlinkedFrom() {
-			i, err := strconv.ParseUint(entry.Link, 16, 64)
-			if err !=  nil {
-				LOG_ERROR("SNAPSHOT_UPLOAD", "Decode error in hardlink entry, expected hex int, got %s", entry.Link)
-				return err
-			}
-
-			targetEntry := hardLinkTable[i].entry
-			var startChunk, endChunk int
-
-			if targetEntry.Size > 0 {
-				startChunk = hardLinkTable[i].startChunk - lastEndChunk
-				endChunk = targetEntry.EndChunk
-			}
-			entry = entry.HardLinkTo(targetEntry, startChunk, endChunk)
-
-			if targetEntry.Size > 0 {
-				lastEndChunk = hardLinkTable[i].startChunk + endChunk
-			}
-
-			LOG_DEBUG("SNAPSHOT_UPLOAD", "Uploading cloned hardlink for %s to %s (%v %v)", entry.Path, targetEntry.Path, startChunk, endChunk)
-		} else if entry.IsHardlinkRoot() {
-			hardLinkTable = append(hardLinkTable, hardLinkEntry{entry, 0})
 		}
 
 		buffer.Reset()

src/duplicacy_entry.go

@@ -4,8 +4,6 @@
 package duplicacy
 
 import (
-	"bytes"
-	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
@@ -17,10 +15,12 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"syscall"
 	"time"
+	"bytes"
+	"crypto/sha256"
+
+    "github.com/vmihailenco/msgpack"
 
-	"github.com/vmihailenco/msgpack"
 )
 
 // This is the hidden directory in the repository for storing various files.
@@ -110,36 +110,15 @@ func (entry *Entry) Copy() *Entry {
 		UID: entry.UID,
 		GID: entry.GID,
 
-		StartChunk:  entry.StartChunk,
+		StartChunk: entry.StartChunk,
 		StartOffset: entry.StartOffset,
-		EndChunk:    entry.EndChunk,
-		EndOffset:   entry.EndOffset,
+		EndChunk: entry.EndChunk,
+		EndOffset: entry.EndOffset,
 
 		Attributes: entry.Attributes,
 	}
 }
 
-func (entry *Entry) HardLinkTo(target *Entry, startChunk int, endChunk int) *Entry {
-	return &Entry{
-		Path: entry.Path,
-		Size: target.Size,
-		Time: target.Time,
-		Mode: target.Mode,
-		Link: entry.Link,
-		Hash: target.Hash,
-
-		UID: target.UID,
-		GID: target.GID,
-
-		StartChunk:  startChunk,
-		StartOffset: target.StartOffset,
-		EndChunk:    endChunk,
-		EndOffset:   target.EndOffset,
-
-		Attributes: target.Attributes,
-	}
-}
-
 // CreateEntryFromJSON creates an entry from a json description.
 func (entry *Entry) UnmarshalJSON(description []byte) (err error) {
 
@@ -383,12 +362,12 @@ func (entry *Entry) EncodeMsgpack(encoder *msgpack.Encoder) error {
 
 	if entry.Attributes != nil {
 		attributes := make([]string, numberOfAttributes)
-		i := 0
-		for attribute := range *entry.Attributes {
-			attributes[i] = attribute
-			i++
-		}
-		sort.Strings(attributes)
+        i := 0
+        for attribute := range *entry.Attributes {
+            attributes[i] = attribute
+            i++
+        }
+        sort.Strings(attributes)
 		for _, attribute := range attributes {
 			err = encoder.EncodeString(attribute)
 			if err != nil {
@@ -401,7 +380,7 @@ func (entry *Entry) EncodeMsgpack(encoder *msgpack.Encoder) error {
 		}
 	}
 
-	return nil
+    return nil
 }
 
 func (entry *Entry) DecodeMsgpack(decoder *msgpack.Decoder) error {
@@ -509,30 +488,18 @@ func (entry *Entry) IsLink() bool {
 	return entry.Mode&uint32(os.ModeSymlink) != 0
 }
 
-func (entry *Entry) IsSpecial() bool {
-	return entry.Mode&uint32(os.ModeNamedPipe|os.ModeDevice|os.ModeCharDevice) != 0
-}
-
 func (entry *Entry) IsComplete() bool {
 	return entry.Size >= 0
 }
 
-func (entry *Entry) IsHardlinkedFrom() bool {
-	return entry.IsFile() && len(entry.Link) > 0 && entry.Link != "/"
-}
-
-func (entry *Entry) IsHardlinkRoot() bool {
-	return entry.IsFile() && entry.Link == "/"
-}
-
 func (entry *Entry) GetPermissions() os.FileMode {
 	return os.FileMode(entry.Mode) & fileModeMask
 }
 
 func (entry *Entry) GetParent() string {
 	path := entry.Path
-	if path != "" && path[len(path)-1] == '/' {
-		path = path[:len(path)-1]
+	if path != "" && path[len(path) - 1] == '/' {
+		path = path[:len(path) - 1]
 	}
 	i := strings.LastIndex(path, "/")
 	if i == -1 {
@@ -629,7 +596,7 @@ func ComparePaths(left string, right string) int {
 	for i := p; i < len(left); i++ {
 		c3 = left[i]
 		if c3 == '/' {
-			last1 = i == len(left)-1
+			last1 = i == len(left) - 1
 			break
 		}
 	}
@@ -639,7 +606,7 @@ func ComparePaths(left string, right string) int {
 	for i := p; i < len(right); i++ {
 		c4 = right[i]
 		if c4 == '/' {
-			last2 = i == len(right)-1
+			last2 = i == len(right) - 1
 			break
 		}
 	}
@@ -727,27 +694,10 @@ func (files FileInfoCompare) Less(i, j int) bool {
 	}
 }
 
-type listEntryLinkKey struct {
-	dev uint64
-	ino uint64
-}
-
-type ListingState struct {
-	linkIndex int
-	linkTable map[listEntryLinkKey]int // map unique inode details to initially found path
-}
-
-func NewListingState() *ListingState {
-	return &ListingState{
-		linkTable: make(map[listEntryLinkKey]int),
-	}
-}
-
 // ListEntries returns a list of entries representing file and subdirectories under the directory 'path'.  Entry paths
 // are normalized as relative to 'top'.  'patterns' are used to exclude or include certain files.
-func ListEntries(top string, path string, patterns []string, nobackupFile string, excludeByAttribute bool,
-	listingState *ListingState,
-	listingChannel chan *Entry) (directoryList []*Entry, skippedFiles []string, err error) {
+func ListEntries(top string, path string, patterns []string, nobackupFile string, excludeByAttribute bool, listingChannel chan *Entry) (directoryList []*Entry,
+	skippedFiles []string, err error) {
 
 	LOG_DEBUG("LIST_ENTRIES", "Listing %s", path)
 
@@ -819,33 +769,6 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
 				}
 				entry = newEntry
 			}
-		} else if entry.Mode & uint32(os.ModeSocket) != 0 {
-			// no reason to issue a warning for what should always be a transient file anyways
-			continue
-		} else if entry.IsSpecial() {
-			if !entry.ReadSpecial(f) {
-				LOG_WARN("LIST_DEV", "Failed to save device node %s", entry.Path)
-				skippedFiles = append(skippedFiles, entry.Path)
-				continue
-			}
-		}
-
-		var linkKey *listEntryLinkKey
-
-		if stat, ok := f.Sys().(*syscall.Stat_t); entry.IsFile() && ok && stat != nil && stat.Nlink > 1 {
-			k := listEntryLinkKey{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}
-			if linkIndex, seen := listingState.linkTable[k]; seen {
-				if linkIndex == -1 {
-					LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute (hardlink)", entry.Path)
-					continue
-				}
-				entry.Size = 0
-				entry.Link = strconv.FormatInt(int64(linkIndex), 16)
-			} else {
-				entry.Link = "/"
-				listingState.linkTable[k] = -1
-				linkKey = &k
-			}
 		}
 
 		entry.ReadAttributes(top)
@@ -855,9 +778,10 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
 			continue
 		}
 
-		if linkKey != nil {
-			listingState.linkTable[*linkKey] = listingState.linkIndex
-			listingState.linkIndex++
+		if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
+			LOG_WARN("LIST_SKIP", "Skipped non-regular file %s", entry.Path)
+			skippedFiles = append(skippedFiles, entry.Path)
+			continue
 		}
 
 		if entry.IsDir() {

src/duplicacy_entrylist.go

@@ -111,12 +111,12 @@ func (entryList *EntryList)createOnDiskFile() error {
 // Add an entry to the entry list
 func (entryList *EntryList)AddEntry(entry *Entry) error {
 
-	if entry.IsFile() {
+	if !entry.IsDir() && !entry.IsLink() {
 		entryList.NumberOfEntries++
 	}
 
 	if !entry.IsComplete() {
-		if !entry.IsFile() {
+		if entry.IsDir() || entry.IsLink() {
 			entry.Size = 0
 		} else {
 			modifiedEntry := ModifiedEntry {

src/duplicacy_snapshot.go

@@ -68,7 +68,6 @@ func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
 								         skippedDirectories *[]string, skippedFiles *[]string) {
 
 	var patterns []string
-	listingState := NewListingState()
 
 	if filtersFile == "" {
 		filtersFile = joinPath(GetDuplicacyPreferencePath(), "filters")
@@ -82,7 +81,7 @@ func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
 
 		directory := directories[len(directories)-1]
 		directories = directories[:len(directories)-1]
-		subdirectories, skipped, err := ListEntries(top, directory.Path, patterns, nobackupFile, excludeByAttribute, listingState, listingChannel)
+		subdirectories, skipped, err := ListEntries(top, directory.Path, patterns, nobackupFile, excludeByAttribute, listingChannel)
 		if err != nil {
 			if directory.Path == "" {
 				LOG_ERROR("LIST_FAILURE", "Failed to list the repository root: %v", err)

src/duplicacy_utils_linux.go

@@ -48,9 +48,6 @@ func ioctl(f *os.File, request uintptr, attrp *uint32) error {
 }
 
 func (entry *Entry) ReadFileFlags(f *os.File) error {
-	if entry.IsSpecial() {
-		return nil
-	}
 	var flags uint32
 	if err := ioctl(f, linux_FS_IOC_GETFLAGS, &flags); err != nil {
 		return err

src/duplicacy_utils_others.go

@@ -49,7 +49,7 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
 
 func (entry *Entry) ReadAttributes(top string) {
 	fullPath := filepath.Join(top, entry.Path)
-	f, err := os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
+	f, err := os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
 	if err != nil {
 		return
 	}
@@ -101,37 +101,6 @@ func (entry *Entry) SetAttributesToFile(fullPath string) {
 	f.Close()
 }
 
-func (entry *Entry) ReadSpecial(fileInfo os.FileInfo) bool {
-	if fileInfo.Mode() & (os.ModeDevice | os.ModeCharDevice) == 0 {
-		return true
-	}
-	stat, ok := fileInfo.Sys().(*syscall.Stat_t)
-	if !ok || stat == nil {
-		return false
-	}
-	entry.Size = 0
-	rdev := uint64(stat.Rdev)
-	entry.StartChunk = int(rdev & 0xFFFFFFFF)
-	entry.StartOffset = int(rdev >> 32)
-	return true
-}
-
-func (entry *Entry) RestoreSpecial(fullPath string) error {
-	if entry.Mode & uint32(os.ModeDevice | os.ModeCharDevice) != 0 {
-		mode := entry.Mode & uint32(fileModeMask)
-		if entry.Mode & uint32(os.ModeCharDevice) != 0 {
-			mode |= syscall.S_IFCHR
-		} else {
-			mode |= syscall.S_IFBLK
-		}
-		rdev := uint64(entry.StartChunk) | uint64(entry.StartOffset) << 32
-		return syscall.Mknod(fullPath, mode, int(rdev))
-	} else if entry.Mode & uint32(os.ModeNamedPipe) != 0 {
-		return syscall.Mkfifo(fullPath, uint32(entry.Mode))
-	}
-	return nil
-}
-
 func joinPath(components ...string) string {
 	return path.Join(components...)
 }

src/duplicacy_utils_windows.go

@@ -117,14 +117,6 @@ func (entry *Entry) SetAttributesToFile(fullPath string) {
 
 }
 
-func (entry *Entry) ReadDeviceNode(fileInfo os.FileInfo) bool {
-	return nil
-}
-
-func (entry *Entry) RestoreSpecial(fullPath string) error {
-	return nil
-}
-
 func joinPath(components ...string) string {
 
 	combinedPath := `\\?\` + filepath.Join(components...)