Compare commits

..

14 Commits
jkl-dev ... jkl

Author SHA1 Message Date
28efe91c3f Support backup of sockets
This only works on Linux. Darwin does not allow mknod of a socket.
Have not tested BSD.
2023-10-04 02:53:48 -05:00
73ca9794ab gofmt some files 2023-10-04 02:53:48 -05:00
70ea4d3acf Fix handling of hardlinks and special files
Also, don't attempt xattr ops on special files on the BSD likes.

TODO: Should have a way to allow restore without special files,
otherwise very cumbersome for a regular user.
2023-10-04 02:53:48 -05:00
a1a3f3d4cb Support for hardlinks to symlinks
This is a specialized use-case, but it is indeed possible to do this
and can be used if xattrs are attached to the symlink.
2023-10-04 02:53:48 -05:00
5b40bf3d93 Fix handling of xattrs with symlinks
Fix Linux, Darwin, and other BSD (untested) to allow proper
handling of xattrs with symlinks. On Linux we cannot use the f*
syscalls for symlinks because symlinks cannot be opened.
File flags must be handled differently on darwin and other BSD due
to the lack of the LCHFLAGS syscall on darwin, and the fact that it
is emulated in libc. However, we do have O_SYMLINK on darwin.
2023-10-04 02:53:48 -05:00
5087ac738d Support backup and restore of special files on POSIX style systems
Special files are device nodes and named pipes. The necessity of the
former is clear, the latter is debatable.
In order to preserve backward compatibility, the device number is
encoded in the StartChunk/StartOffset fields of the entry.
2023-10-04 02:53:48 -05:00
99e4dcae00 Don't overwrite symlinks if file already exists 2023-10-04 02:53:48 -05:00
997ff5bdf6 Support backup and restore of hardlinks
This tracks inode/device from the stat info and creates backward
compatible snapshots that allow preserving hardlinks. Backwards
compatibility is preserved by saving a virtual inode number index in the
Link field of the file entry. Since this field was previously only used
for symlinks, this won't break old versions. Additionally, the entry
data is cloned so restoration with an old version works.

Current limitations are primarility with restore. They include:
- no command line option to prevent hard link restore
- if a file has the immutable or append only flag it will be set before
hardlinks are restored, so hardlinking will fail.
- if a partial restore includes a hardlink but not the parent
directories the hardlink will fail.

These will be solved by grouping restore of hardlinks together
with file, prior to applying final metadata.

- if a file is changed and is being rewritten by a restore hardlinks are
not preserved.
2023-10-04 02:53:48 -05:00
34b1e19278 Initial implementation of file/inode flags (Linux, BSD, darwin)
Basic support for BSD and Darwin style chflags (stat flags). Applies
these flags at the end of file restore.
Supports linux style ioctl_iflags(2) in a 2 step process. Flags that
need to be applied prior to writes such as compress and especially no-COW
are applied immediately upon file open.

The flags format is backwards compatible. An attribute starting with a
null byte is used to store flags in the entry attributes table. With
an old version of duplicacy the restore of this attribute should silently
fail (effectively be ignored).

Fixes xattr restore to use O_NOFOLLOW so attributes are applied to symlink.

TODO: Tests, possible option to switch off mutable/append prior to
restore of existing file similar to rsync. Does not apply attributes
or flags to the top most directory.
2023-10-04 02:53:48 -05:00
c1c8af1de9 Increase b2 client max file listing count to 10000
Considerable speed improvement with listing large storage.
2023-10-04 02:53:48 -05:00
8b788572c8 Fix exclude_by_attribute feature on POSIX
The exclude by attribute function is broken on non-Darwin POSIX: linux and freebsd.
This is because those xattrs must be prefixed by a legal namespace. The old xattr
library implicitly appended the user namespace to the xattr, but the current
official go pkg does not (which is just as well).

Also fix the test to remove the discordant old xattr dependency and provide
test cases for both darwin and non-darwin POSIX.
2023-10-04 02:53:48 -05:00
b592484b54 Use S3 ListObjectsV2 for listing files
ListObjects has been deprecated since 2016 and ListObjectsV2 with use of
explicit pagination tokens is more performant for large listings as well.

This also mitigates an issue with iDrive E2 where the StartAfter/Marker
is included in the output, leading to duplicate entries. Right now this
causes an exhaustive prune to delete chunks erroneously flagged as
duplicate, destroying the storage.
2023-10-04 02:53:48 -05:00
Gilbert Chen
50120146df Bump version to 3.2.2 2023-10-03 22:35:18 -04:00
Gilbert Chen
7bfc0e7d51 Save passwords after a storj storage has been created 2023-10-03 22:33:41 -04:00
6 changed files with 36 additions and 20 deletions

View File

@@ -2262,7 +2262,7 @@ func main() {
app.Name = "duplicacy" app.Name = "duplicacy"
app.HelpName = "duplicacy" app.HelpName = "duplicacy"
app.Usage = "A new generation cloud backup tool based on lock-free deduplication" app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
app.Version = "3.2.1" + " (" + GitCommit + ")" app.Version = "3.2.2" + " (" + GitCommit + ")"
// Exit with code 2 if an invalid command is provided // Exit with code 2 if an invalid command is provided
app.CommandNotFound = func(context *cli.Context, command string) { app.CommandNotFound = func(context *cli.Context, command string) {

View File

@@ -516,7 +516,7 @@ func (entry *Entry) IsLink() bool {
} }
func (entry *Entry) IsSpecial() bool { func (entry *Entry) IsSpecial() bool {
return entry.Mode&uint32(os.ModeNamedPipe|os.ModeDevice|os.ModeCharDevice) != 0 return entry.Mode&uint32(os.ModeNamedPipe|os.ModeDevice|os.ModeCharDevice|os.ModeSocket) != 0
} }
func (entry *Entry) IsFileOrSpecial() bool { func (entry *Entry) IsFileOrSpecial() bool {
@@ -807,9 +807,6 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
if f.Name() == DUPLICACY_DIRECTORY { if f.Name() == DUPLICACY_DIRECTORY {
continue continue
} }
if f.Mode()&os.ModeSocket != 0 {
continue
}
entry := CreateEntryFromFileInfo(f, normalizedPath) entry := CreateEntryFromFileInfo(f, normalizedPath)
if len(patterns) > 0 && !MatchPath(entry.Path, patterns) { if len(patterns) > 0 && !MatchPath(entry.Path, patterns) {

View File

@@ -756,6 +756,8 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
LOG_ERROR("STORAGE_CREATE", "Failed to load the Storj storage at %s: %v", storageURL, err) LOG_ERROR("STORAGE_CREATE", "Failed to load the Storj storage at %s: %v", storageURL, err)
return nil return nil
} }
SavePassword(preference, "storj_key", apiKey)
SavePassword(preference, "storj_passphrase", passphrase)
return storjStorage return storjStorage
} else if matched[1] == "smb" { } else if matched[1] == "smb" {
server := matched[3] server := matched[3]

View File

@@ -92,3 +92,18 @@ func (entry *Entry) RestoreEarlyDirFlags(path string) error {
func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error { func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
return nil return nil
} }
func (entry *Entry) RestoreSpecial(fullPath string) error {
mode := entry.Mode & uint32(fileModeMask)
if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
mode |= syscall.S_IFIFO
} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
mode |= syscall.S_IFCHR
} else if entry.Mode&uint32(os.ModeDevice) != 0 {
mode |= syscall.S_IFBLK
} else {
return nil
}
return syscall.Mknod(fullPath, mode, int(entry.GetRdev()))
}

View File

@@ -219,6 +219,23 @@ func (entry *Entry) restoreLateFileFlags(f *os.File) error {
return nil return nil
} }
func (entry *Entry) RestoreSpecial(fullPath string) error {
mode := entry.Mode & uint32(fileModeMask)
if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
mode |= syscall.S_IFIFO
} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
mode |= syscall.S_IFCHR
} else if entry.Mode&uint32(os.ModeDevice) != 0 {
mode |= syscall.S_IFBLK
} else if entry.Mode&uint32(os.ModeSocket) != 0 {
mode |= syscall.S_IFSOCK
} else {
return nil
}
return syscall.Mknod(fullPath, mode, int(entry.GetRdev()))
}
func excludedByAttribute(attributes map[string][]byte) bool { func excludedByAttribute(attributes map[string][]byte) bool {
_, ok := attributes["user.duplicacy_exclude"] _, ok := attributes["user.duplicacy_exclude"]
return ok return ok

View File

@@ -65,21 +65,6 @@ func (entry *Entry) GetRdev() uint64 {
return uint64(entry.StartChunk) | uint64(entry.StartOffset)<<32 return uint64(entry.StartChunk) | uint64(entry.StartOffset)<<32
} }
func (entry *Entry) RestoreSpecial(fullPath string) error {
mode := entry.Mode & uint32(fileModeMask)
if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
mode |= syscall.S_IFIFO
} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
mode |= syscall.S_IFCHR
} else if entry.Mode&uint32(os.ModeDevice) != 0 {
mode |= syscall.S_IFBLK
} else {
return nil
}
return syscall.Mknod(fullPath, mode, int(entry.GetRdev()))
}
func (entry *Entry) IsSameSpecial(fileInfo os.FileInfo) bool { func (entry *Entry) IsSameSpecial(fileInfo os.FileInfo) bool {
stat := fileInfo.Sys().(*syscall.Stat_t) stat := fileInfo.Sys().(*syscall.Stat_t)
if stat == nil { if stat == nil {