Support backup and restore of special files on POSIX style systems

Special files are device nodes and named pipes. The necessity of the
former is clear, the latter is debatable.
In order to preserve backward compatibility, the device number is
encoded in the StartChunk/StartOffset fields of the entry.

src/duplicacy_backupmanager.go

@@ -797,6 +797,21 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 			}
 			remoteEntry.RestoreEarlyDirFlags(fullPath)
 			directoryEntries = append(directoryEntries, remoteEntry)
+		} else if remoteEntry.IsSpecial() {
+			if stat, _ := os.Lstat(fullPath); stat != nil {
+				if !overwrite {
+					LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
+						"File %s already exists.  Please specify the -overwrite option to overwrite", remoteEntry.Path)
+					continue
+				}
+				os.Remove(fullPath)
+			}
+
+			if err := remoteEntry.RestoreSpecial(fullPath); err != nil {
+				LOG_ERROR("RESTORE_SPECIAL", "Unable to restore special file %s: %v", remoteEntry.Path, err)
+				return 0
+			}
+			remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
 		} else {
 			if remoteEntry.IsHardlinkRoot() {
 				hardLinkTable[len(hardLinkTable)-1] = hardLinkEntry{remoteEntry, true}

src/duplicacy_entry.go

@@ -509,6 +509,10 @@ func (entry *Entry) IsLink() bool {
 	return entry.Mode&uint32(os.ModeSymlink) != 0
 }
 
+func (entry *Entry) IsSpecial() bool {
+	return entry.Mode&uint32(os.ModeNamedPipe|os.ModeDevice|os.ModeCharDevice) != 0
+}
+
 func (entry *Entry) IsComplete() bool {
 	return entry.Size >= 0
 }
@@ -815,13 +819,16 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
 				}
 				entry = newEntry
 			}
-		}
+		} else if entry.Mode & uint32(os.ModeSocket) != 0 {
-
+			// no reason to issue a warning for what should always be a transient file anyways
-		if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
+			continue
-			LOG_WARN("LIST_SKIP", "Skipped non-regular file %s", entry.Path)
+		} else if entry.IsSpecial() {
+			if !entry.ReadSpecial(f) {
+				LOG_WARN("LIST_DEV", "Failed to save device node %s", entry.Path)
 				skippedFiles = append(skippedFiles, entry.Path)
 				continue
 			}
+		}
 
 		var linkKey *listEntryLinkKey
 

src/duplicacy_entrylist.go

@@ -111,12 +111,12 @@ func (entryList *EntryList)createOnDiskFile() error {
 // Add an entry to the entry list
 func (entryList *EntryList)AddEntry(entry *Entry) error {
 
-	if !entry.IsDir() && !entry.IsLink() {
+	if entry.IsFile() {
 		entryList.NumberOfEntries++
 	}
 
 	if !entry.IsComplete() {
-		if entry.IsDir() || entry.IsLink() {
+		if !entry.IsFile() {
 			entry.Size = 0
 		} else {
 			modifiedEntry := ModifiedEntry {

src/duplicacy_utils_bsd.go

@@ -0,0 +1,53 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+//go:build freebsd || netbsd || darwin
+// +build freebsd netbsd darwin
+
+package duplicacy
+
+import (
+	"encoding/binary"
+	"os"
+	"syscall"
+)
+
+const bsdFileFlagsKey = "\x00bf"
+
+func (entry *Entry) ReadFileFlags(f *os.File) error {
+	fileInfo, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	stat, ok := fileInfo.Sys().(*syscall.Stat_t)
+	if ok && stat.Flags != 0 {
+		if entry.Attributes == nil {
+			entry.Attributes = &map[string][]byte{}
+		}
+		v := make([]byte, 4)
+		binary.LittleEndian.PutUint32(v, stat.Flags)
+		(*entry.Attributes)[bsdFileFlagsKey] = v
+		LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", stat.Flags, entry.Path)
+	}
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyDirFlags(path string) error {
+	return nil
+}
+
+func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
+	return nil
+}
+
+func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
+	if entry.Attributes == nil {
+		return nil
+	}
+	if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
+		LOG_DEBUG("ATTR_RESTORE", "Restore flags 0x%x for %s", binary.LittleEndian.Uint32(v), entry.Path)
+		return syscall.Fchflags(int(f.Fd()), int(binary.LittleEndian.Uint32(v)))
+	}
+	return nil
+}

src/duplicacy_utils_bsd_common.go

@@ -1,90 +0,0 @@
-// Copyright (c) Acrosync LLC. All rights reserved.
-// Free for personal use and commercial trial
-// Commercial use requires per-user licenses available from https://duplicacy.com
-
-//go:build freebsd || netbsd || darwin
-// +build freebsd netbsd darwin
-
-package duplicacy
-
-import (
-	"encoding/binary"
-	"os"
-	"path/filepath"
-	"bytes"
-	"syscall"
-
-	"github.com/pkg/xattr"
-)
-
-const bsdFileFlagsKey = "\x00bf"
-
-func (entry *Entry) ReadAttributes(top string) {
-	fullPath := filepath.Join(top, entry.Path)
-	fileInfo, err := os.Lstat(fullPath)
-	if err != nil {
-		return
-	}
-	attributes, _ := xattr.LList(fullPath)
-	if len(attributes) > 0 {
-		entry.Attributes = &map[string][]byte{}
-		for _, name := range attributes {
-			attribute, err := xattr.LGet(fullPath, name)
-			if err == nil {
-				(*entry.Attributes)[name] = attribute
-			}
-		}
-	}
-	if err := entry.readFileFlags(fileInfo); err != nil {
-		LOG_INFO("ATTR_BACKUP", "Could not backup flags for file %s: %v", fullPath, err)
-	}
-}
-
-func (entry *Entry) SetAttributesToFile(fullPath string) {
-	names, _ := xattr.LList(fullPath)
-	for _, name := range names {
-		newAttribute, found := (*entry.Attributes)[name]
-		if found {
-			oldAttribute, _ := xattr.LGet(fullPath, name)
-			if !bytes.Equal(oldAttribute, newAttribute) {
-				xattr.LSet(fullPath, name, newAttribute)
-			}
-			delete(*entry.Attributes, name)
-		} else {
-			xattr.LRemove(fullPath, name)
-		}
-	}
-
-	for name, attribute := range *entry.Attributes {
-		if len(name) > 0 && name[0] == '\x00' {
-			continue
-		}
-		xattr.LSet(fullPath, name, attribute)
-	}
-	if err := entry.restoreLateFileFlags(fullPath); err != nil {
-		LOG_DEBUG("ATTR_RESTORE", "Could not restore flags for file %s: %v", fullPath, err)
-	}
-}
-
-func (entry *Entry) readFileFlags(fileInfo os.FileInfo) error {
-	stat, ok := fileInfo.Sys().(*syscall.Stat_t)
-	if ok && stat.Flags != 0 {
-		if entry.Attributes == nil {
-			entry.Attributes = &map[string][]byte{}
-		}
-		v := make([]byte, 4)
-		binary.LittleEndian.PutUint32(v, stat.Flags)
-		(*entry.Attributes)[bsdFileFlagsKey] = v
-		LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", stat.Flags, entry.Path)
-	}
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyDirFlags(path string) error {
-	return nil
-}
-
-func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
-	return nil
-}
-

src/duplicacy_utils_darwin.go

@@ -5,29 +5,10 @@
 package duplicacy
 
 import (
-	"os"
-	"syscall"
 	"strings"
-	"encoding/binary"
 )
 
 func excludedByAttribute(attributes map[string][]byte) bool {
 	value, ok := attributes["com.apple.metadata:com_apple_backup_excludeItem"]
 	return ok && strings.Contains(string(value), "com.apple.backupd")
 }
-
-func (entry *Entry) restoreLateFileFlags(path string) error {
-	if entry.Attributes == nil {
-		return nil
-	}
-	if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
-		f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_SYMLINK, 0)
-		if err != nil {
-			return err
-		}
-		err = syscall.Fchflags(int(f.Fd()), int(binary.LittleEndian.Uint32(v)))
-		f.Close()
-		return err
-	}
-	return nil
-}

src/duplicacy_utils_linux.go

@@ -5,14 +5,10 @@
 package duplicacy
 
 import (
-	"bytes"
 	"encoding/binary"
 	"os"
 	"syscall"
 	"unsafe"
-	"path/filepath"
-
-	"github.com/pkg/xattr"
 )
 
 const (
@@ -51,113 +47,10 @@ func ioctl(f *os.File, request uintptr, attrp *uint32) error {
 	return nil
 }
 
-type xattrHandle struct {
+func (entry *Entry) ReadFileFlags(f *os.File) error {
-	f *os.File
+	if entry.IsSpecial() {
-	fullPath string
+		return nil
 	}
-
-func (x xattrHandle) list() ([]string, error) {
-	if x.f != nil {
-		return xattr.FList(x.f)
-	} else {
-		return xattr.LList(x.fullPath)
-	}
-}
-
-func (x xattrHandle) get(name string) ([]byte, error) {
-	if x.f != nil {
-		return xattr.FGet(x.f, name)
-	} else {
-		return xattr.LGet(x.fullPath, name)
-	}
-}
-
-func (x xattrHandle) set(name string, value []byte) error {
-	if x.f != nil {
-		return xattr.FSet(x.f, name, value)
-	} else {
-		return xattr.LSet(x.fullPath, name, value)
-	}
-}
-
-func (x xattrHandle) remove(name string) error {
-	if x.f != nil {
-		return xattr.FRemove(x.f, name)
-	} else {
-		return xattr.LSet(x.fullPath, name)
-	}
-}
-
-func (entry *Entry) ReadAttributes(top string) {
-	x := xattrHandle{nil, filepath.Join(top, entry.Path)}
-
-	if !entry.IsLink() {
-		x.f, err := os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
-		if err != nil {
-			// FIXME: We really should return errors for failure to read
-			return
-		}
-	}
-
-	attributes, _ := x.list()
-
-	if len(attributes) > 0 {
-		entry.Attributes = &map[string][]byte{}
-	}
-	for _, name := range attributes {
-		attribute, err := x.get(f, name)
-		if err == nil {
-			(*entry.Attributes)[name] = attribute
-		}
-	}
-
-	if entry.IsFile() || entry.IsDir() {
-		if err := entry.readFileFlags(x.f); err != nil {
-			LOG_INFO("ATTR_BACKUP", "Could not backup flags for file %s: %v", fullPath, err)
-		}
-	}
-	f.Close()
-}
-
-func (entry *Entry) SetAttributesToFile(fullPath string) {
-	x := xattrHandle{nil, fullPath}
-	if !entry.IsLink() {
-		x.f, err := os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
-		if err != nil {
-			return
-		}
-	}
-
-	names, _ := x.list()
-
-	for _, name := range names {
-		newAttribute, found := (*entry.Attributes)[name]
-		if found {
-			oldAttribute, _ := x.get(name)
-			if !bytes.Equal(oldAttribute, newAttribute) {
-				x.set(name, newAttribute)
-			}
-			delete(*entry.Attributes, name)
-		} else {
-			x.remove(name)
-		}
-	}
-
-	for name, attribute := range *entry.Attributes {
-		if len(name) > 0 && name[0] == '\x00' {
-			continue
-		}
-		x.set(name, attribute)
-	}
-	if entry.IsFile() || entry.IsDir() {
-		if err := entry.restoreLateFileFlags(f); err != nil {
-			LOG_DEBUG("ATTR_RESTORE", "Could not restore flags for file %s: %v", fullPath, err)
-		}
-	}
-	f.Close()
-}
-
-func (entry *Entry) readFileFlags(f *os.File) error {
 	var flags uint32
 	if err := ioctl(f, linux_FS_IOC_GETFLAGS, &flags); err != nil {
 		return err
@@ -204,7 +97,7 @@ func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
 	return nil
 }
 
-func (entry *Entry) restoreLateFileFlags(f *os.File) error {
+func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
 	if entry.Attributes == nil {
 		return nil
 	}

src/duplicacy_utils_others.go

@@ -2,15 +2,18 @@
 // Free for personal use and commercial trial
 // Commercial use requires per-user licenses available from https://duplicacy.com
 
-//go:build !windows
 // +build !windows
 
 package duplicacy
 
 import (
+	"bytes"
 	"os"
 	"path"
+	"path/filepath"
 	"syscall"
+
+	"github.com/pkg/xattr"
 )
 
 func Readlink(path string) (isRegular bool, s string, err error) {
@@ -44,6 +47,91 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
 	return true
 }
 
+func (entry *Entry) ReadAttributes(top string) {
+	fullPath := filepath.Join(top, entry.Path)
+	f, err := os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
+	if err != nil {
+		return
+	}
+	attributes, _ := xattr.FList(f)
+	if len(attributes) > 0 {
+		entry.Attributes = &map[string][]byte{}
+		for _, name := range attributes {
+			attribute, err := xattr.Get(fullPath, name)
+			if err == nil {
+				(*entry.Attributes)[name] = attribute
+			}
+		}
+	}
+	if err := entry.ReadFileFlags(f); err != nil {
+		LOG_INFO("ATTR_BACKUP", "Could not backup flags for file %s: %v", fullPath, err)
+	}
+	f.Close()
+}
+
+func (entry *Entry) SetAttributesToFile(fullPath string) {
+	f, err := os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
+	if err != nil {
+		return
+	}
+
+	names, _ := xattr.FList(f)
+	for _, name := range names {
+		newAttribute, found := (*entry.Attributes)[name]
+		if found {
+			oldAttribute, _ := xattr.FGet(f, name)
+			if !bytes.Equal(oldAttribute, newAttribute) {
+				xattr.FSet(f, name, newAttribute)
+			}
+			delete(*entry.Attributes, name)
+		} else {
+			xattr.FRemove(f, name)
+		}
+	}
+
+	for name, attribute := range *entry.Attributes {
+		if len(name) > 0 && name[0] == '\x00' {
+			continue
+		}
+		xattr.FSet(f, name, attribute)
+	}
+	if err := entry.RestoreLateFileFlags(f); err != nil {
+		LOG_DEBUG("ATTR_RESTORE", "Could not restore flags for file %s: %v", fullPath, err)
+	}
+	f.Close()
+}
+
+func (entry *Entry) ReadSpecial(fileInfo os.FileInfo) bool {
+	if fileInfo.Mode() & (os.ModeDevice | os.ModeCharDevice) == 0 {
+		return true
+	}
+	stat, ok := fileInfo.Sys().(*syscall.Stat_t)
+	if !ok || stat == nil {
+		return false
+	}
+	entry.Size = 0
+	rdev := uint64(stat.Rdev)
+	entry.StartChunk = int(rdev & 0xFFFFFFFF)
+	entry.StartOffset = int(rdev >> 32)
+	return true
+}
+
+func (entry *Entry) RestoreSpecial(fullPath string) error {
+	if entry.Mode & uint32(os.ModeDevice | os.ModeCharDevice) != 0 {
+		mode := entry.Mode & uint32(fileModeMask)
+		if entry.Mode & uint32(os.ModeCharDevice) != 0 {
+			mode |= syscall.S_IFCHR
+		} else {
+			mode |= syscall.S_IFBLK
+		}
+		rdev := uint64(entry.StartChunk) | uint64(entry.StartOffset) << 32
+		return syscall.Mknod(fullPath, mode, int(rdev))
+	} else if entry.Mode & uint32(os.ModeNamedPipe) != 0 {
+		return syscall.Mkfifo(fullPath, uint32(entry.Mode))
+	}
+	return nil
+}
+
 func joinPath(components ...string) string {
 	return path.Join(components...)
 }

src/duplicacy_utils_windows.go

@@ -117,6 +117,14 @@ func (entry *Entry) SetAttributesToFile(fullPath string) {
 
 }
 
+func (entry *Entry) ReadDeviceNode(fileInfo os.FileInfo) bool {
+	return nil
+}
+
+func (entry *Entry) RestoreSpecial(fullPath string) error {
+	return nil
+}
+
 func joinPath(components ...string) string {
 
 	combinedPath := `\\?\` + filepath.Join(components...)

src/duplicacy_utils_xbsd.go

@@ -1,30 +0,0 @@
-// Copyright (c) Acrosync LLC. All rights reserved.
-// Free for personal use and commercial trial
-// Commercial use requires per-user licenses available from https://duplicacy.com
-
-//go:build freebsd || netbsd
-// +build freebsd netbsd
-
-package duplicacy
-
-import (
-	"bytes"
-	"encoding/binary"
-	"os"
-	"path/filepath"
-	"syscall"
-
-	"github.com/pkg/xattr"
-)
-
-func (entry *Entry) restoreLateFileFlags(path string) error {
-	if entry.Attributes == nil {
-		return nil
-	}
-	if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
-		if _, _, errno := syscall.Syscall(syscall.SYS_LCHFLAGS, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(v), 0); errno != 0 {
-			return os.NewSyscallError("lchflags", errno)
-		}
-	}
-	return nil
-}