Compare commits

..

2 Commits

Author SHA1 Message Date
Gilbert Chen
50120146df Bump version to 3.2.2 2023-10-03 22:35:18 -04:00
Gilbert Chen
7bfc0e7d51 Save passwords after a storj storage has been created 2023-10-03 22:33:41 -04:00
19 changed files with 162 additions and 844 deletions

View File

@@ -2262,7 +2262,7 @@ func main() {
app.Name = "duplicacy"
app.HelpName = "duplicacy"
app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
app.Version = "3.2.1" + " (" + GitCommit + ")"
app.Version = "3.2.2" + " (" + GitCommit + ")"
// Exit with code 2 if an invalid command is provided
app.CommandNotFound = func(context *cli.Context, command string) {

1
go.mod
View File

@@ -15,6 +15,7 @@ require (
github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c
github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8
github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508
github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01
github.com/hirochachacha/go-smb2 v1.1.0
github.com/klauspost/compress v1.16.3
github.com/klauspost/reedsolomon v1.9.9

2
go.sum
View File

@@ -47,6 +47,8 @@ github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8 h1:ijgl4Y+
github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8/go.mod h1:0lQcVva56+L1PuUFXLOsJ6arJQaU0baIH8q+IegeBhg=
github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508 h1:SqTyk5KkNXp7zTdTttIZSDcTrL5uau4K/2OpKvgBZVI=
github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508/go.mod h1:w/pisxUZezf2XzU9Ewjphcf6q1mZtOzKPHhJiuc8cag=
github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01 h1:LqwS9qL6SrDkp0g0iwUkETrDdtB9gTKaIbSn9imUq5o=
github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01/go.mod h1:TMlibuxKfkdtHyltooAw7+DHqRpaXs9nxaffk00Sh1Q=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 h1:G1U0vew/vA/1/hBmf1XNeyIzJJbPFVv+kb+HPl6rj6c=

View File

@@ -396,7 +396,7 @@ type B2ListFileNamesOutput struct {
func (client *B2Client) ListFileNames(threadIndex int, startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
maxFileCount := 10_000
maxFileCount := 1000
if singleFile {
if includeVersions {
maxFileCount = 4

View File

@@ -187,7 +187,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
// If the listing operation is fast and this is an initial backup, list all chunks and
// put them in the cache.
if manager.storage.IsFastListing() && remoteSnapshot.Revision == 0 {
if (manager.storage.IsFastListing() && remoteSnapshot.Revision == 0) {
LOG_INFO("BACKUP_LIST", "Listing all chunks")
allChunks, _ := manager.SnapshotManager.ListAllFiles(manager.storage, "chunks/")
@@ -703,42 +703,8 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
var localEntry *Entry
localListingOK := true
type hardLinkEntry struct {
entry *Entry
willExist bool
}
var hardLinkTable []hardLinkEntry
var hardLinks []*Entry
restoreHardlink := func(entry *Entry, fullPath string) bool {
if entry.IsHardlinkRoot() {
hardLinkTable[len(hardLinkTable)-1].willExist = true
} else if entry.IsHardlinkedFrom() {
i, err := entry.GetHardlinkId()
if err != nil {
LOG_ERROR("RESTORE_HARDLINK", "Decode error for hardlinked entry %s, %v", entry.Path, err)
return false
}
if !hardLinkTable[i].willExist {
hardLinkTable[i] = hardLinkEntry{entry, true}
} else {
sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
LOG_INFO("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath)
if err := MakeHardlink(sourcePath, fullPath); err != nil {
LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s %v", fullPath, sourcePath, err)
}
return true
}
}
return false
}
for remoteEntry := range remoteListingChannel {
if remoteEntry.IsHardlinkRoot() {
hardLinkTable = append(hardLinkTable, hardLinkEntry{remoteEntry, false})
}
if len(patterns) > 0 && !MatchPath(remoteEntry.Path, patterns) {
continue
}
@@ -775,39 +741,26 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
}
fullPath := joinPath(top, remoteEntry.Path)
if remoteEntry.IsLink() {
if stat, _ := os.Lstat(fullPath); stat != nil {
stat, err := os.Lstat(fullPath)
if stat != nil {
if stat.Mode()&os.ModeSymlink != 0 {
isRegular, link, err := Readlink(fullPath)
if err == nil && link == remoteEntry.Link && !isRegular {
remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
if remoteEntry.IsHardlinkRoot() {
hardLinkTable[len(hardLinkTable)-1].willExist = true
}
continue
}
}
if !overwrite {
LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
"File %s already exists. Please specify the -overwrite option to overwrite", remoteEntry.Path)
continue
}
os.Remove(fullPath)
}
if restoreHardlink(remoteEntry, fullPath) {
continue
}
if err := os.Symlink(remoteEntry.Link, fullPath); err != nil {
err = os.Symlink(remoteEntry.Link, fullPath)
if err != nil {
LOG_ERROR("RESTORE_SYMLINK", "Can't create symlink %s: %v", remoteEntry.Path, err)
return 0
}
remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
LOG_TRACE("DOWNLOAD_DONE", "Symlink %s updated", remoteEntry.Path)
} else if remoteEntry.IsDir() {
@@ -827,50 +780,8 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
return 0
}
}
remoteEntry.RestoreEarlyDirFlags(fullPath)
directoryEntries = append(directoryEntries, remoteEntry)
} else if remoteEntry.IsSpecial() {
if stat, _ := os.Lstat(fullPath); stat != nil {
if remoteEntry.IsSameSpecial(stat) {
remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
if remoteEntry.IsHardlinkRoot() {
hardLinkTable[len(hardLinkTable)-1].willExist = true
}
}
if !overwrite {
LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
"File %s already exists. Please specify the -overwrite option to overwrite", remoteEntry.Path)
continue
}
os.Remove(fullPath)
}
if restoreHardlink(remoteEntry, fullPath) {
continue
}
if err := remoteEntry.RestoreSpecial(fullPath); err != nil {
LOG_ERROR("RESTORE_SPECIAL", "Unable to restore special file %s: %v", remoteEntry.Path, err)
return 0
}
remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
} else {
if remoteEntry.IsHardlinkRoot() {
hardLinkTable[len(hardLinkTable)-1].willExist = true
} else if remoteEntry.IsHardlinkedFrom() {
i, err := remoteEntry.GetHardlinkId()
if err != nil {
LOG_ERROR("RESTORE_HARDLINK", "Decode error for hardlinked entry %s, %v", remoteEntry.Path, err)
return 0
}
if !hardLinkTable[i].willExist {
hardLinkTable[i] = hardLinkEntry{remoteEntry, true}
} else {
hardLinks = append(hardLinks, remoteEntry)
continue
}
}
// We can't download files here since fileEntries needs to be sorted
fileEntries = append(fileEntries, remoteEntry)
totalFileSize += remoteEntry.Size
@@ -990,38 +901,6 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
file.RestoreMetadata(fullPath, nil, setOwner)
}
for _, linkEntry := range hardLinks {
i, _ := linkEntry.GetHardlinkId()
sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
fullPath := joinPath(top, linkEntry.Path)
if stat, _ := os.Lstat(fullPath); stat != nil {
sourceStat, _ := os.Lstat(sourcePath)
if os.SameFile(stat, sourceStat) {
continue
}
if sourceStat == nil {
LOG_WERROR(allowFailures, "RESTORE_HARDLINK",
"Target %s for hardlink %s is missing", sourcePath, linkEntry.Path)
continue
}
if !overwrite {
LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
"File %s already exists. Please specify the -overwrite option to overwrite", linkEntry.Path)
continue
}
os.Remove(fullPath)
}
LOG_DEBUG("RESTORE_HARDLINK", "Hard linking %s to %s", fullPath, sourcePath)
if err := MakeHardlink(sourcePath, fullPath); err != nil {
LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s", fullPath, sourcePath)
return 0
}
}
if deleteMode && len(patterns) == 0 {
// Reverse the order to make sure directories are empty before being deleted
for i := range extraFiles {
@@ -1173,13 +1052,8 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
lastEndChunk := 0
type hardLinkEntry struct {
entry *Entry
startChunk int
}
var hardLinkTable []hardLinkEntry
uploadEntryInfoFunc := func(entry *Entry) error {
if entry.IsFile() && entry.Size > 0 {
delta := entry.StartChunk - len(chunkHashes) + 1
if entry.StartChunk != lastChunk {
@@ -1197,38 +1071,10 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
entry.StartChunk -= delta
entry.EndChunk -= delta
if entry.IsHardlinkRoot() {
LOG_DEBUG("SNAPSHOT_UPLOAD", "Hard link root %s %v %v", entry.Path, entry.StartChunk, entry.EndChunk)
hardLinkTable = append(hardLinkTable, hardLinkEntry{entry, entry.StartChunk})
}
delta = entry.EndChunk - entry.StartChunk
entry.StartChunk -= lastEndChunk
lastEndChunk = entry.EndChunk
entry.EndChunk = delta
} else if entry.IsHardlinkedFrom() && !entry.IsLink() {
i, err := entry.GetHardlinkId()
if err != nil {
LOG_ERROR("SNAPSHOT_UPLOAD", "Decode error for hardlinked entry %s, %v", entry.Link, err)
return err
}
targetEntry := hardLinkTable[i].entry
var startChunk, endChunk int
if targetEntry.Size > 0 {
startChunk = hardLinkTable[i].startChunk - lastEndChunk
endChunk = targetEntry.EndChunk
}
entry = entry.HardLinkTo(targetEntry, startChunk, endChunk)
if targetEntry.Size > 0 {
lastEndChunk = hardLinkTable[i].startChunk + endChunk
}
LOG_DEBUG("SNAPSHOT_UPLOAD", "Uploading cloned hardlink for %s to %s (%v %v)", entry.Path, targetEntry.Path, startChunk, endChunk)
} else if entry.IsHardlinkRoot() {
hardLinkTable = append(hardLinkTable, hardLinkEntry{entry, 0})
}
buffer.Reset()
@@ -1298,7 +1144,6 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
// file under the .duplicacy directory and then replaces the existing one. Otherwise, the existing file will be
// overwritten directly.
// Return: true, nil: Restored file;
//
// false, nil: Skipped file;
// false, error: Failure to restore file (only if allowFailures == true)
func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chunkMaker *ChunkMaker, entry *Entry, top string, inPlace bool, overwrite bool,
@@ -1349,7 +1194,6 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing: %v", fullPath, err)
return false, nil
}
entry.RestoreEarlyFileFlags(existingFile)
n := int64(1)
// There is a go bug on Windows (https://github.com/golang/go/issues/21681) that causes Seek to fail
@@ -1533,7 +1377,6 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
return false, nil
}
}
entry.RestoreEarlyFileFlags(existingFile)
existingFile.Seek(0, 0)
@@ -1616,7 +1459,6 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
LOG_ERROR("DOWNLOAD_OPEN", "Failed to open file for writing: %v", err)
return false, nil
}
entry.RestoreEarlyFileFlags(newFile)
hasher := manager.config.NewFileHasher()

View File

@@ -4,11 +4,8 @@
package duplicacy
import (
"bytes"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
@@ -18,15 +15,12 @@ import (
"sort"
"strconv"
"strings"
"syscall"
"time"
"bytes"
"crypto/sha256"
"github.com/vmihailenco/msgpack"
)
const (
entrySymHardLinkRootChunkMarker = -72
entrySymHardLinkTargetChunkMarker = -73
)
// This is the hidden directory in the repository for storing various files.
@@ -125,27 +119,6 @@ func (entry *Entry) Copy() *Entry {
}
}
func (entry *Entry) HardLinkTo(target *Entry, startChunk int, endChunk int) *Entry {
return &Entry{
Path: entry.Path,
Size: target.Size,
Time: target.Time,
Mode: target.Mode,
Link: entry.Link,
Hash: target.Hash,
UID: target.UID,
GID: target.GID,
StartChunk: startChunk,
StartOffset: target.StartOffset,
EndChunk: endChunk,
EndOffset: target.EndOffset,
Attributes: target.Attributes,
}
}
// CreateEntryFromJSON creates an entry from a json description.
func (entry *Entry) UnmarshalJSON(description []byte) (err error) {
@@ -515,38 +488,10 @@ func (entry *Entry) IsLink() bool {
return entry.Mode&uint32(os.ModeSymlink) != 0
}
func (entry *Entry) IsSpecial() bool {
return entry.Mode&uint32(os.ModeNamedPipe|os.ModeDevice|os.ModeCharDevice) != 0
}
func (entry *Entry) IsFileOrSpecial() bool {
return entry.Mode&uint32(os.ModeDir|os.ModeSymlink|os.ModeIrregular) == 0
}
func (entry *Entry) IsComplete() bool {
return entry.Size >= 0
}
func (entry *Entry) IsHardlinkedFrom() bool {
return (entry.IsFileOrSpecial() && len(entry.Link) > 0 && entry.Link != "/") || (entry.IsLink() && entry.StartChunk == entrySymHardLinkTargetChunkMarker)
}
func (entry *Entry) IsHardlinkRoot() bool {
return (entry.IsFileOrSpecial() && entry.Link == "/") || (entry.IsLink() && entry.StartChunk == entrySymHardLinkRootChunkMarker)
}
func (entry *Entry) GetHardlinkId() (int, error) {
if entry.IsLink() {
if entry.StartChunk != entrySymHardLinkTargetChunkMarker {
return 0, errors.New("Symlink entry not marked as hardlinked")
}
return entry.StartOffset, nil
} else {
i, err := strconv.ParseUint(entry.Link, 16, 64)
return int(i), err
}
}
func (entry *Entry) GetPermissions() os.FileMode {
return os.FileMode(entry.Mode) & fileModeMask
}
@@ -605,10 +550,6 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setO
}
}
if entry.Attributes != nil && len(*entry.Attributes) > 0 {
entry.SetAttributesToFile(fullPath)
}
// Only set the time if the file is not a symlink
if !entry.IsLink() && (*fileInfo).ModTime().Unix() != entry.Time {
modifiedTime := time.Unix(entry.Time, 0)
@@ -619,6 +560,10 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setO
}
}
if entry.Attributes != nil && len(*entry.Attributes) > 0 {
entry.SetAttributesToFile(fullPath)
}
return true
}
@@ -749,27 +694,10 @@ func (files FileInfoCompare) Less(i, j int) bool {
}
}
type listEntryLinkKey struct {
dev uint64
ino uint64
}
type ListingState struct {
linkIndex int
linkTable map[listEntryLinkKey]int // map unique inode details to initially found path
}
func NewListingState() *ListingState {
return &ListingState{
linkTable: make(map[listEntryLinkKey]int),
}
}
// ListEntries returns a list of entries representing file and subdirectories under the directory 'path'. Entry paths
// are normalized as relative to 'top'. 'patterns' are used to exclude or include certain files.
func ListEntries(top string, path string, patterns []string, nobackupFile string, excludeByAttribute bool,
listingState *ListingState,
listingChannel chan *Entry) (directoryList []*Entry, skippedFiles []string, err error) {
func ListEntries(top string, path string, patterns []string, nobackupFile string, excludeByAttribute bool, listingChannel chan *Entry) (directoryList []*Entry,
skippedFiles []string, err error) {
LOG_DEBUG("LIST_ENTRIES", "Listing %s", path)
@@ -807,44 +735,10 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
if f.Name() == DUPLICACY_DIRECTORY {
continue
}
if f.Mode()&os.ModeSocket != 0 {
continue
}
entry := CreateEntryFromFileInfo(f, normalizedPath)
if len(patterns) > 0 && !MatchPath(entry.Path, patterns) {
continue
}
var linkKey *listEntryLinkKey
if runtime.GOOS != "windows" && !entry.IsDir() {
if stat := f.Sys().(*syscall.Stat_t); stat != nil && stat.Nlink > 1 {
k := listEntryLinkKey{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}
if linkIndex, seen := listingState.linkTable[k]; seen {
if linkIndex == -1 {
LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute (hardlink)", entry.Path)
continue
}
entry.Size = 0
if entry.IsLink() {
entry.StartChunk = entrySymHardLinkTargetChunkMarker
entry.StartOffset = linkIndex
} else {
entry.Link = strconv.FormatInt(int64(linkIndex), 16)
}
} else {
if entry.IsLink() {
entry.StartChunk = entrySymHardLinkRootChunkMarker
} else {
entry.Link = "/"
}
listingState.linkTable[k] = -1
linkKey = &k
}
}
}
if entry.IsLink() {
isRegular := false
isRegular, entry.Link, err = Readlink(joinPath(top, entry.Path))
@@ -875,12 +769,6 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
}
entry = newEntry
}
} else if entry.IsSpecial() {
if !entry.ReadSpecial(f) {
LOG_WARN("LIST_DEV", "Failed to save device node %s", entry.Path)
skippedFiles = append(skippedFiles, entry.Path)
continue
}
}
entry.ReadAttributes(top)
@@ -890,9 +778,10 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
continue
}
if linkKey != nil {
listingState.linkTable[*linkKey] = listingState.linkIndex
listingState.linkIndex++
if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
LOG_WARN("LIST_SKIP", "Skipped non-regular file %s", entry.Path)
skippedFiles = append(skippedFiles, entry.Path)
continue
}
if entry.IsDir() {

View File

@@ -5,8 +5,6 @@
package duplicacy
import (
"bytes"
"encoding/json"
"io/ioutil"
"math/rand"
"os"
@@ -15,9 +13,10 @@ import (
"sort"
"strings"
"testing"
"bytes"
"encoding/json"
"github.com/pkg/xattr"
"github.com/gilbertchen/xattr"
"github.com/vmihailenco/msgpack"
)
@@ -234,16 +233,8 @@ func TestEntryOrder(t *testing.T) {
// TestEntryExcludeByAttribute tests the excludeByAttribute parameter to the ListEntries function
func TestEntryExcludeByAttribute(t *testing.T) {
var excludeAttrName string
var excludeAttrValue []byte
if runtime.GOOS == "darwin" {
excludeAttrName = "com.apple.metadata:com_apple_backup_excludeItem"
excludeAttrValue = []byte("com.apple.backupd")
} else if runtime.GOOS == "linux" || runtime.GOOS == "freebsd" || runtime.GOOS == "netbsd" || runtime.GOOS == "solaris" {
excludeAttrName = "user.duplicacy_exclude"
} else {
t.Skip("skipping test, not darwin, linux, freebsd, netbsd, or solaris")
if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") {
t.Skip("skipping test not darwin or linux")
}
testDir := filepath.Join(os.TempDir(), "duplicacy_test")
@@ -282,7 +273,7 @@ func TestEntryExcludeByAttribute(t *testing.T) {
for _, file := range DATA {
fullPath := filepath.Join(testDir, file)
if strings.Contains(file, "exclude") {
xattr.Set(fullPath, excludeAttrName, excludeAttrValue)
xattr.Setxattr(fullPath, "com.apple.metadata:com_apple_backup_excludeItem", []byte("com.apple.backupd"))
}
}

View File

@@ -111,12 +111,12 @@ func (entryList *EntryList)createOnDiskFile() error {
// Add an entry to the entry list
func (entryList *EntryList)AddEntry(entry *Entry) error {
if entry.IsFile() {
if !entry.IsDir() && !entry.IsLink() {
entryList.NumberOfEntries++
}
if !entry.IsComplete() {
if !entry.IsFile() {
if entry.IsDir() || entry.IsLink() {
entry.Size = 0
} else {
modifiedEntry := ModifiedEntry {

View File

@@ -90,40 +90,48 @@ func (storage *S3Storage) ListFiles(threadIndex int, dir string) (files []string
if dir == "snapshots/" {
dir = storage.storageDir + dir
input := s3.ListObjectsV2Input{
input := s3.ListObjectsInput{
Bucket: aws.String(storage.bucket),
Prefix: aws.String(dir),
Delimiter: aws.String("/"),
}
err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
for _, subDir := range page.CommonPrefixes {
files = append(files, (*subDir.Prefix)[len(dir):])
}
return true
})
if err != nil {
return nil, nil, err
}
return files, nil, nil
} else {
dir = storage.storageDir + dir
input := s3.ListObjectsV2Input{
Bucket: aws.String(storage.bucket),
Prefix: aws.String(dir),
MaxKeys: aws.Int64(1000),
}
err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
for _, object := range page.Contents {
output, err := storage.client.ListObjects(&input)
if err != nil {
return nil, nil, err
}
for _, subDir := range output.CommonPrefixes {
files = append(files, (*subDir.Prefix)[len(dir):])
}
return files, nil, nil
} else {
dir = storage.storageDir + dir
marker := ""
for {
input := s3.ListObjectsInput{
Bucket: aws.String(storage.bucket),
Prefix: aws.String(dir),
MaxKeys: aws.Int64(1000),
Marker: aws.String(marker),
}
output, err := storage.client.ListObjects(&input)
if err != nil {
return nil, nil, err
}
for _, object := range output.Contents {
files = append(files, (*object.Key)[len(dir):])
sizes = append(sizes, *object.Size)
}
return true
})
if err != nil {
return nil, nil, err
if !*output.IsTruncated {
break
}
marker = *output.Contents[len(output.Contents)-1].Key
}
return files, sizes, nil
}

View File

@@ -68,7 +68,6 @@ func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
skippedDirectories *[]string, skippedFiles *[]string) {
var patterns []string
listingState := NewListingState()
if filtersFile == "" {
filtersFile = joinPath(GetDuplicacyPreferencePath(), "filters")
@@ -82,7 +81,7 @@ func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
directory := directories[len(directories)-1]
directories = directories[:len(directories)-1]
subdirectories, skipped, err := ListEntries(top, directory.Path, patterns, nobackupFile, excludeByAttribute, listingState, listingChannel)
subdirectories, skipped, err := ListEntries(top, directory.Path, patterns, nobackupFile, excludeByAttribute, listingChannel)
if err != nil {
if directory.Path == "" {
LOG_ERROR("LIST_FAILURE", "Failed to list the repository root: %v", err)

View File

@@ -756,6 +756,8 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
LOG_ERROR("STORAGE_CREATE", "Failed to load the Storj storage at %s: %v", storageURL, err)
return nil
}
SavePassword(preference, "storj_key", apiKey)
SavePassword(preference, "storj_passphrase", passphrase)
return storjStorage
} else if matched[1] == "smb" {
server := matched[3]

View File

@@ -7,15 +7,14 @@ package duplicacy
import (
"bufio"
"crypto/sha256"
"encoding/json"
"fmt"
"io"
"os"
"regexp"
"runtime"
"strconv"
"strings"
"time"
"runtime"
"github.com/gilbertchen/gopass"
"golang.org/x/crypto/pbkdf2"
@@ -276,6 +275,7 @@ func SavePassword(preference Preference, passwordType string, password string) {
// The following code was modified from the online article 'Matching Wildcards: An Algorithm', by Kirk J. Krauss,
// Dr. Dobb's, August 26, 2008. However, the version in the article doesn't handle cases like matching 'abcccd'
// against '*ccd', and the version here fixed that issue.
//
func matchPattern(text string, pattern string) bool {
textLength := len(text)
@@ -474,34 +474,3 @@ func PrintMemoryUsage() {
time.Sleep(time.Second)
}
}
func (entry *Entry) dump() map[string]interface{} {
object := make(map[string]interface{})
object["path"] = entry.Path
object["size"] = entry.Size
object["time"] = entry.Time
object["mode"] = entry.Mode
object["hash"] = entry.Hash
object["link"] = entry.Link
object["content"] = fmt.Sprintf("%d:%d:%d:%d",
entry.StartChunk, entry.StartOffset, entry.EndChunk, entry.EndOffset)
if entry.UID != -1 && entry.GID != -1 {
object["uid"] = entry.UID
object["gid"] = entry.GID
}
if entry.Attributes != nil && len(*entry.Attributes) > 0 {
object["attributes"] = entry.Attributes
}
return object
}
func (entry *Entry) dumpString() string {
data, _ := json.Marshal(entry.dump())
return string(data)
}

View File

@@ -1,94 +0,0 @@
// Copyright (c) Acrosync LLC. All rights reserved.
// Free for personal use and commercial trial
// Commercial use requires per-user licenses available from https://duplicacy.com
//go:build freebsd || netbsd || darwin
// +build freebsd netbsd darwin
package duplicacy
import (
"bytes"
"encoding/binary"
"os"
"path/filepath"
"syscall"
"github.com/pkg/xattr"
)
const bsdFileFlagsKey = "\x00bf"
func (entry *Entry) ReadAttributes(top string) {
fullPath := filepath.Join(top, entry.Path)
fileInfo, err := os.Lstat(fullPath)
if err != nil {
return
}
if !entry.IsSpecial() {
attributes, _ := xattr.LList(fullPath)
if len(attributes) > 0 {
entry.Attributes = &map[string][]byte{}
for _, name := range attributes {
attribute, err := xattr.LGet(fullPath, name)
if err == nil {
(*entry.Attributes)[name] = attribute
}
}
}
}
if err := entry.readFileFlags(fileInfo); err != nil {
LOG_INFO("ATTR_BACKUP", "Could not backup flags for file %s: %v", fullPath, err)
}
}
func (entry *Entry) SetAttributesToFile(fullPath string) {
if !entry.IsSpecial() {
names, _ := xattr.LList(fullPath)
for _, name := range names {
newAttribute, found := (*entry.Attributes)[name]
if found {
oldAttribute, _ := xattr.LGet(fullPath, name)
if !bytes.Equal(oldAttribute, newAttribute) {
xattr.LSet(fullPath, name, newAttribute)
}
delete(*entry.Attributes, name)
} else {
xattr.LRemove(fullPath, name)
}
}
for name, attribute := range *entry.Attributes {
if len(name) > 0 && name[0] == '\x00' {
continue
}
xattr.LSet(fullPath, name, attribute)
}
}
if err := entry.restoreLateFileFlags(fullPath); err != nil {
LOG_DEBUG("ATTR_RESTORE", "Could not restore flags for file %s: %v", fullPath, err)
}
}
func (entry *Entry) readFileFlags(fileInfo os.FileInfo) error {
stat, ok := fileInfo.Sys().(*syscall.Stat_t)
if ok && stat.Flags != 0 {
if entry.Attributes == nil {
entry.Attributes = &map[string][]byte{}
}
v := make([]byte, 4)
binary.LittleEndian.PutUint32(v, stat.Flags)
(*entry.Attributes)[bsdFileFlagsKey] = v
LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", stat.Flags, entry.Path)
}
return nil
}
func (entry *Entry) RestoreEarlyDirFlags(path string) error {
return nil
}
func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
return nil
}

View File

@@ -5,29 +5,10 @@
package duplicacy
import (
"encoding/binary"
"os"
"strings"
"syscall"
)
func excludedByAttribute(attributes map[string][]byte) bool {
value, ok := attributes["com.apple.metadata:com_apple_backup_excludeItem"]
func excludedByAttribute(attirbutes map[string][]byte) bool {
value, ok := attirbutes["com.apple.metadata:com_apple_backup_excludeItem"]
return ok && strings.Contains(string(value), "com.apple.backupd")
}
func (entry *Entry) restoreLateFileFlags(path string) error {
if entry.Attributes == nil {
return nil
}
if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_SYMLINK, 0)
if err != nil {
return err
}
err = syscall.Fchflags(int(f.Fd()), int(binary.LittleEndian.Uint32(v)))
f.Close()
return err
}
return nil
}

View File

@@ -2,12 +2,12 @@
// Free for personal use and commercial trial
// Commercial use requires per-user licenses available from https://duplicacy.com
//go:build freebsd || netbsd || solaris
// +build freebsd netbsd solaris
package duplicacy
func excludedByAttribute(attributes map[string][]byte) bool {
_, ok := attributes["user.duplicacy_exclude"]
import (
)
func excludedByAttribute(attirbutes map[string][]byte) bool {
_, ok := attirbutes["duplicacy_exclude"]
return ok
}

View File

@@ -5,221 +5,9 @@
package duplicacy
import (
"bytes"
"encoding/binary"
"os"
"path/filepath"
"syscall"
"unsafe"
"github.com/pkg/xattr"
)
const (
linux_FS_SECRM_FL = 0x00000001 /* Secure deletion */
linux_FS_UNRM_FL = 0x00000002 /* Undelete */
linux_FS_COMPR_FL = 0x00000004 /* Compress file */
linux_FS_SYNC_FL = 0x00000008 /* Synchronous updates */
linux_FS_IMMUTABLE_FL = 0x00000010 /* Immutable file */
linux_FS_APPEND_FL = 0x00000020 /* writes to file may only append */
linux_FS_NODUMP_FL = 0x00000040 /* do not dump file */
linux_FS_NOATIME_FL = 0x00000080 /* do not update atime */
linux_FS_NOCOMP_FL = 0x00000400 /* Don't compress */
linux_FS_JOURNAL_DATA_FL = 0x00004000 /* Reserved for ext3 */
linux_FS_NOTAIL_FL = 0x00008000 /* file tail should not be merged */
linux_FS_DIRSYNC_FL = 0x00010000 /* dirsync behaviour (directories only) */
linux_FS_TOPDIR_FL = 0x00020000 /* Top of directory hierarchies*/
linux_FS_NOCOW_FL = 0x00800000 /* Do not cow file */
linux_FS_PROJINHERIT_FL = 0x20000000 /* Create with parents projid */
linux_FS_IOC_GETFLAGS uintptr = 0x80086601
linux_FS_IOC_SETFLAGS uintptr = 0x40086602
linuxIocFlagsFileEarly = linux_FS_SECRM_FL | linux_FS_UNRM_FL | linux_FS_COMPR_FL | linux_FS_NODUMP_FL | linux_FS_NOATIME_FL | linux_FS_NOCOMP_FL | linux_FS_JOURNAL_DATA_FL | linux_FS_NOTAIL_FL | linux_FS_NOCOW_FL
linuxIocFlagsDirEarly = linux_FS_TOPDIR_FL | linux_FS_PROJINHERIT_FL
linuxIocFlagsLate = linux_FS_SYNC_FL | linux_FS_IMMUTABLE_FL | linux_FS_APPEND_FL | linux_FS_DIRSYNC_FL
linuxFileFlagsKey = "\x00lf"
)
func ioctl(f *os.File, request uintptr, attrp *uint32) error {
argp := uintptr(unsafe.Pointer(attrp))
if _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), request, argp); errno != 0 {
return os.NewSyscallError("ioctl", errno)
}
return nil
}
type xattrHandle struct {
f *os.File
fullPath string
}
func (x xattrHandle) list() ([]string, error) {
if x.f != nil {
return xattr.FList(x.f)
} else {
return xattr.LList(x.fullPath)
}
}
func (x xattrHandle) get(name string) ([]byte, error) {
if x.f != nil {
return xattr.FGet(x.f, name)
} else {
return xattr.LGet(x.fullPath, name)
}
}
func (x xattrHandle) set(name string, value []byte) error {
if x.f != nil {
return xattr.FSet(x.f, name, value)
} else {
return xattr.LSet(x.fullPath, name, value)
}
}
func (x xattrHandle) remove(name string) error {
if x.f != nil {
return xattr.FRemove(x.f, name)
} else {
return xattr.LRemove(x.fullPath, name)
}
}
func (entry *Entry) ReadAttributes(top string) {
fullPath := filepath.Join(top, entry.Path)
x := xattrHandle{nil, fullPath}
if !entry.IsLink() {
var err error
x.f, err = os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
if err != nil {
// FIXME: We really should return errors for failure to read
return
}
}
attributes, _ := x.list()
if len(attributes) > 0 {
entry.Attributes = &map[string][]byte{}
}
for _, name := range attributes {
attribute, err := x.get(name)
if err == nil {
(*entry.Attributes)[name] = attribute
}
}
if entry.IsFile() || entry.IsDir() {
if err := entry.readFileFlags(x.f); err != nil {
LOG_INFO("ATTR_BACKUP", "Could not backup flags for file %s: %v", fullPath, err)
}
}
x.f.Close()
}
func (entry *Entry) SetAttributesToFile(fullPath string) {
x := xattrHandle{nil, fullPath}
if !entry.IsLink() {
var err error
x.f, err = os.OpenFile(fullPath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
if err != nil {
return
}
}
names, _ := x.list()
for _, name := range names {
newAttribute, found := (*entry.Attributes)[name]
if found {
oldAttribute, _ := x.get(name)
if !bytes.Equal(oldAttribute, newAttribute) {
x.set(name, newAttribute)
}
delete(*entry.Attributes, name)
} else {
x.remove(name)
}
}
for name, attribute := range *entry.Attributes {
if len(name) > 0 && name[0] == '\x00' {
continue
}
x.set(name, attribute)
}
if entry.IsFile() || entry.IsDir() {
if err := entry.restoreLateFileFlags(x.f); err != nil {
LOG_DEBUG("ATTR_RESTORE", "Could not restore flags for file %s: %v", fullPath, err)
}
}
x.f.Close()
}
func (entry *Entry) readFileFlags(f *os.File) error {
var flags uint32
if err := ioctl(f, linux_FS_IOC_GETFLAGS, &flags); err != nil {
return err
}
if flags != 0 {
if entry.Attributes == nil {
entry.Attributes = &map[string][]byte{}
}
v := make([]byte, 4)
binary.LittleEndian.PutUint32(v, flags)
(*entry.Attributes)[linuxFileFlagsKey] = v
LOG_DEBUG("ATTR_READ", "Read flags 0x%x for %s", flags, entry.Path)
}
return nil
}
func (entry *Entry) RestoreEarlyDirFlags(path string) error {
if entry.Attributes == nil {
return nil
}
if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
flags := binary.LittleEndian.Uint32(v) & linuxIocFlagsDirEarly
f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_DIRECTORY, 0)
if err != nil {
return err
}
LOG_DEBUG("ATTR_RESTORE", "Restore dir flags (early) 0x%x for %s", flags, entry.Path)
err = ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
f.Close()
return err
}
return nil
}
func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
if entry.Attributes == nil {
return nil
}
if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
flags := binary.LittleEndian.Uint32(v) & linuxIocFlagsFileEarly
LOG_DEBUG("ATTR_RESTORE", "Restore flags (early) 0x%x for %s", flags, entry.Path)
return ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
}
return nil
}
func (entry *Entry) restoreLateFileFlags(f *os.File) error {
if entry.Attributes == nil {
return nil
}
if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
flags := binary.LittleEndian.Uint32(v) & (linuxIocFlagsFileEarly | linuxIocFlagsDirEarly | linuxIocFlagsLate)
LOG_DEBUG("ATTR_RESTORE", "Restore flags (late) 0x%x for %s", flags, entry.Path)
return ioctl(f, linux_FS_IOC_SETFLAGS, &flags)
}
return nil
}
func excludedByAttribute(attributes map[string][]byte) bool {
_, ok := attributes["user.duplicacy_exclude"]
func excludedByAttribute(attirbutes map[string][]byte) bool {
_, ok := attirbutes["duplicacy_exclude"]
return ok
}

View File

@@ -2,17 +2,18 @@
// Free for personal use and commercial trial
// Commercial use requires per-user licenses available from https://duplicacy.com
//go:build !windows
// +build !windows
package duplicacy
import (
"bytes"
"os"
"path"
"path/filepath"
"syscall"
"golang.org/x/sys/unix"
"github.com/pkg/xattr"
)
func Readlink(path string) (isRegular bool, s string, err error) {
@@ -46,50 +47,43 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
return true
}
func (entry *Entry) ReadSpecial(fileInfo os.FileInfo) bool {
if fileInfo.Mode()&(os.ModeDevice|os.ModeCharDevice) == 0 {
return true
func (entry *Entry) ReadAttributes(top string) {
fullPath := filepath.Join(top, entry.Path)
attributes, _ := xattr.List(fullPath)
if len(attributes) > 0 {
entry.Attributes = &map[string][]byte{}
for _, name := range attributes {
attribute, err := xattr.Get(fullPath, name)
if err == nil {
(*entry.Attributes)[name] = attribute
}
}
stat := fileInfo.Sys().(*syscall.Stat_t)
if stat == nil {
return false
}
entry.Size = 0
rdev := uint64(stat.Rdev)
entry.StartChunk = int(rdev & 0xFFFFFFFF)
entry.StartOffset = int(rdev >> 32)
return true
}
func (entry *Entry) GetRdev() uint64 {
return uint64(entry.StartChunk) | uint64(entry.StartOffset)<<32
func (entry *Entry) SetAttributesToFile(fullPath string) {
names, _ := xattr.List(fullPath)
for _, name := range names {
newAttribute, found := (*entry.Attributes)[name]
if found {
oldAttribute, _ := xattr.Get(fullPath, name)
if !bytes.Equal(oldAttribute, newAttribute) {
xattr.Set(fullPath, name, newAttribute)
}
func (entry *Entry) RestoreSpecial(fullPath string) error {
mode := entry.Mode & uint32(fileModeMask)
if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
mode |= syscall.S_IFIFO
} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
mode |= syscall.S_IFCHR
} else if entry.Mode&uint32(os.ModeDevice) != 0 {
mode |= syscall.S_IFBLK
delete(*entry.Attributes, name)
} else {
return nil
xattr.Remove(fullPath, name)
}
return syscall.Mknod(fullPath, mode, int(entry.GetRdev()))
}
func (entry *Entry) IsSameSpecial(fileInfo os.FileInfo) bool {
stat := fileInfo.Sys().(*syscall.Stat_t)
if stat == nil {
return false
}
return (uint32(fileInfo.Mode()) == entry.Mode) && (uint64(stat.Rdev) == entry.GetRdev())
for name, attribute := range *entry.Attributes {
xattr.Set(fullPath, name, attribute)
}
func MakeHardlink(source string, target string) error {
return unix.Linkat(unix.AT_FDCWD, source, unix.AT_FDCWD, target, 0)
}
func joinPath(components ...string) string {

View File

@@ -117,18 +117,6 @@ func (entry *Entry) SetAttributesToFile(fullPath string) {
}
func (entry *Entry) ReadDeviceNode(fileInfo os.FileInfo) bool {
return nil
}
func (entry *Entry) RestoreSpecial(fullPath string) error {
return nil
}
func MakeHardlink(source string, target string) error {
return os.Link(source, target)
}
func joinPath(components ...string) string {
combinedPath := `\\?\` + filepath.Join(components...)
@@ -144,18 +132,6 @@ func SplitDir(fullPath string) (dir string, file string) {
return fullPath[:i+1], fullPath[i+1:]
}
func (entry *Entry) ReadFileFlags(f *os.File) error {
return nil
}
func (entry *Entry) RestoreEarlyDirFlags(path string) error {
return nil
}
func (entry *Entry) RestoreEarlyFileFlags(f *os.File) error {
return nil
}
func (entry *Entry) RestoreLateFileFlags(f *os.File) error {
return nil
func excludedByAttribute(attirbutes map[string][]byte) bool {
return false
}

View File

@@ -1,30 +0,0 @@
// Copyright (c) Acrosync LLC. All rights reserved.
// Free for personal use and commercial trial
// Commercial use requires per-user licenses available from https://duplicacy.com
//go:build freebsd || netbsd
// +build freebsd netbsd
package duplicacy
import (
"bytes"
"encoding/binary"
"os"
"path/filepath"
"syscall"
"github.com/pkg/xattr"
)
func (entry *Entry) restoreLateFileFlags(path string) error {
if entry.Attributes == nil {
return nil
}
if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
if _, _, errno := syscall.Syscall(syscall.SYS_LCHFLAGS, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(v), 0); errno != 0 {
return os.NewSyscallError("lchflags", errno)
}
}
return nil
}