mirror of
https://github.com/jkl1337/duplicacy.git
synced 2026-01-02 03:34:39 -06:00
Compare commits
75 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 518d02a57d | |||
| 58d21eb17a | |||
| c151b21f5c | |||
| 6d5cb4b7b9 | |||
| 09637c69bc | |||
| bede63f33f | |||
| 95063913ae | |||
| 3481814562 | |||
| 9ff5484ec0 | |||
| a33bf0a21b | |||
| 228ca7005c | |||
| 23a3cce0ad | |||
| 8427c03b71 | |||
| af89dd5e7b | |||
| a479fa5197 | |||
| 986e6a6ef2 | |||
| 756a003fb3 | |||
| 07c796a46b | |||
| f24e8d2cbb | |||
| cfff4ff425 | |||
| 6c1600e8c8 | |||
| 807268d337 | |||
| ab8fc0faa6 | |||
| 8cd457447b | |||
| ee19d44403 | |||
| b8c0b1f86e | |||
| 059e74da17 | |||
| 31c131b9e3 | |||
| e01008a56b | |||
| 4934df4e08 | |||
| 52e628ac93 | |||
| 88c92da9bd | |||
| fee1cb859e | |||
| e61f5c2192 | |||
| d3b7cdc02d | |||
| 0bfab89057 | |||
| 3468117fe1 | |||
| b443e05113 | |||
|
2549532676 | ||
|
|
e99dfea048 | ||
|
|
50120146df | ||
|
|
7bfc0e7d51 | ||
|
|
fd3bceae19 | ||
|
|
7cc1b4222c | ||
|
|
d92b1734f4 | ||
|
|
4e9d2c4cca | ||
|
|
cc482beb95 | ||
|
bf3ea8a83c | ||
|
|
981efc13e6 | ||
|
|
6445ecbcde | ||
|
|
ff207ba5bf | ||
|
|
3a81c1065a | ||
|
|
cdf8f5a857 | ||
|
|
1f9ad0e35c | ||
|
|
53b0f3f7b6 | ||
|
|
9f276047db | ||
|
|
c237269589 | ||
|
|
493ef603e3 | ||
|
889191a814 | ||
|
|
df80096cdf | ||
|
|
24c2ea76b9 | ||
|
|
15b6ef9d76 | ||
|
|
75b310b98e | ||
|
039b749a3e | ||
|
|
9be475f876 | ||
|
|
27ff3e216b | ||
|
|
1ba204a21b | ||
|
|
b8c7594dbf | ||
|
|
58f0d2be5a | ||
|
|
0a794e6fea | ||
|
|
bc2d762e41 | ||
|
|
6a7a2c8048 | ||
|
|
3472206bcf | ||
|
d7593a828c | ||
|
|
238ef63e16 |
42
README.md
42
README.md
@@ -1,3 +1,45 @@
|
||||
# Dupluxe
|
||||
|
||||
An experimental Duplicacy derivative with improved support for preserving state on UNIX like systems. Produces snapshots compatible with Duplicacy.
|
||||
|
||||
NOTE: This project/repository is not affiliated with nor endorsed by Duplicacy, Acrosync or their associated rights holders. This project is open source but is not free/libre software. It is developed and distributed in accordance with the associated LICENSE. Commercial use may require purchase of a license from Acrosync, please contact them if you have any doubts.
|
||||
|
||||
## Added Features
|
||||
* Support for hard links. Hard links are tracked during local file listing. All linked entries will reuse the same chunk data, so this can give a time and space saving benefit as hard-linked files only need to be packed once. Hard links are supported to everything (regular files, symlinks, special files) except directories.
|
||||
* Optional File flags, that is chflags(1) on BSD/Darwin, and ioctl_iflags(2) on Linux. The primary use case is to preserve iflags used by btrfs for no-COW and compression.
|
||||
* Optional Special files (character/block devices, FIFOs, and sockets) are preserved along with associated metadata.
|
||||
|
||||
## Assorted Changes
|
||||
* The S3 backend uses the newer ListObjectsV2 interface originally because of a bug with some providers with the old, obsolete interface, but now because this API is considerably faster on a number of providers tested.
|
||||
* B2 client max listing per request increased to 10,000
|
||||
* A fix for the exclude_by_attribute feature on BSD/Unix which has been broken upstream for ages.
|
||||
|
||||
## Snapshot Format
|
||||
The generated preserves snapshots are backward compatible with vanilla versions of duplicacy and also do not increase the encoding size of metadata significantly. Unfortunately duplicacy does not have a formal forward-compatible snapshot versioning system, but that's not too surprising. This does mean that the data encoding is somewhat abusive of the existing format.
|
||||
|
||||
### Hard links
|
||||
The storage differs for regular files vs. every other target. Entry records contain a `Link` string field for the symlink target. When a likely hard linked file is encountered (`st_nlink > 1`) that entry is marked as a hard link root with the string "/" in the `Link` field and it is placed in an array, the index of this array will serve as a link address. Plain duplicacy only uses the `Link` field for symlinks. Files that hard link to this initial file have the index of the array of root files encoded as a base-16 integer into their `Link` field. These entries are placed in the snapshot with valid start/end chunk and offset values and all metadata is cloned so official Duplicacy will recover them as regular files with all metadata, it just will never make hard links.
|
||||
For hard links to symlinks and special files, the `Link` isn't used. Instead, since these files never have content the `EndChunk/EndOffset` fields are used. A magic number (-9) is encoded in `EndChunk` for root entries and (-10) for clone/child entries. The `EndOffset` contains the index into the root entry array.
|
||||
|
||||
### Special Files
|
||||
Duplicacy simply skips special files. DuplicacyIX does not skip them. The `st_rdev` (device number) for character and block devices is stored with the lower 32-bits in `StartChunk` and the upper 32-bits in `StartOffset`, though no actual supported system uses anything bigger than 32-bits. The packing of this quantity is OS specific, but major, minor numbers are also OS specific.
|
||||
|
||||
### File flags
|
||||
Files flags are stored in the extended attributes table with a short (2 character) OS specific key prefixed with a null-byte. Duplicacy will try to set these xattrs, however they will be ignored as the name appears to be empty with the initial null-byte.
|
||||
|
||||
|
||||
## Motivation
|
||||
Arguably system root directories are better preserved in a filesystem image format, however the line becomes blurred for home and data directories the former which tends to become a magnet for all kinds of data layout. This gives the option of a convenient random addressable cloud backup with easy partial restore while also being able to backup an nearly exact replica for use in disaster recovery. Nearly exact, the only metadata not preserved are times other than mtimes and ACLs on BSD-like systems.
|
||||
Hard links are a pain and might be better to not exist but in actual use, but things like git repos and SDKs have a tendency to use them. Often one has no choice but to deal with them, and forgoing preserving them is painful.
|
||||
File flags are primarily for the use case of btrfs snapshot backups, specifically with regards to compression and no-COW. The implementation applies certain flags immediately on open so that these flags apply to written blocks.
|
||||
Special files serve a couple purposes. Backup of FIFOs and sockets are primarily for preserving metadata since these files have no useful content and can always be created on the fly. The other is support for backup of overlay2 file systems. overlay2 uses character mode dev-nodes for whiteouts in addition to trusted namespace xattrs. DuplicacyIX should be able to faithfully reproduce overlay2 fs layers.
|
||||
|
||||
## Caveats/TODO
|
||||
* Improve command line options around processing hard links and special files, especially for restore. Right now if you don't have `mknod` capabilities and the snapshot has special device files you're going to need to use filters.
|
||||
* File flags for immutability aren't handled smartly. Specifically immutable and append only files will break badly with hardlinks, since hardlink creation is deferred to after flags application. The solution is not complicated but this is not a pressing use case.
|
||||
* Some corner cases of replacing existing files with hard links might end up breaking links if not doing a full restore. Again not a pressing use case. For the primary use of disaster recovery of large portions or an entire volume it works fine.
|
||||
* Possibly encode ACLs on Mac OS/FreeBSD. On Linux the crappy POSIX 1e ACLs that no one should use are picked up in the xattrs.
|
||||
|
||||
# Duplicacy: A lock-free deduplication cloud backup tool
|
||||
|
||||
Duplicacy is a new generation cross-platform cloud backup tool based on the idea of [Lock-Free Deduplication](https://github.com/gilbertchen/duplicacy/wiki/Lock-Free-Deduplication).
|
||||
|
||||
@@ -22,9 +22,7 @@ import (
|
||||
|
||||
"github.com/gilbertchen/cli"
|
||||
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/gilbertchen/duplicacy/src"
|
||||
duplicacy "github.com/gilbertchen/duplicacy/src"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -316,7 +314,7 @@ func configRepository(context *cli.Context, init bool) {
|
||||
// write real path into .duplicacy file inside repository
|
||||
duplicacyFileName := path.Join(repository, duplicacy.DUPLICACY_FILE)
|
||||
d1 := []byte(preferencePath)
|
||||
err = ioutil.WriteFile(duplicacyFileName, d1, 0644)
|
||||
err = os.WriteFile(duplicacyFileName, d1, 0644)
|
||||
if err != nil {
|
||||
duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to write %s file inside repository %v", duplicacyFileName, err)
|
||||
return
|
||||
@@ -368,9 +366,8 @@ func configRepository(context *cli.Context, init bool) {
|
||||
"The storage '%s' has already been initialized", preference.StorageURL)
|
||||
if existingConfig.CompressionLevel >= -1 && existingConfig.CompressionLevel <= 9 {
|
||||
duplicacy.LOG_INFO("STORAGE_FORMAT", "This storage is configured to use the pre-1.2.0 format")
|
||||
} else if existingConfig.CompressionLevel != 100 {
|
||||
duplicacy.LOG_ERROR("STORAGE_COMPRESSION", "This storage is configured with an invalid compression level %d", existingConfig.CompressionLevel)
|
||||
return
|
||||
} else if existingConfig.CompressionLevel != duplicacy.DEFAULT_COMPRESSION_LEVEL {
|
||||
duplicacy.LOG_INFO("STORAGE_COMPRESSION", "Compression level: %d", existingConfig.CompressionLevel)
|
||||
}
|
||||
|
||||
// Don't print config in the background mode
|
||||
@@ -378,8 +375,6 @@ func configRepository(context *cli.Context, init bool) {
|
||||
existingConfig.Print()
|
||||
}
|
||||
} else {
|
||||
compressionLevel := 100
|
||||
|
||||
averageChunkSize := duplicacy.AtoSize(context.String("chunk-size"))
|
||||
if averageChunkSize == 0 {
|
||||
fmt.Fprintf(context.App.Writer, "Invalid average chunk size: %s.\n\n", context.String("chunk-size"))
|
||||
@@ -487,6 +482,18 @@ func configRepository(context *cli.Context, init bool) {
|
||||
}
|
||||
}
|
||||
|
||||
compressionLevel := 100
|
||||
zstdLevel := context.String("zstd-level")
|
||||
if zstdLevel != "" {
|
||||
if level, found := duplicacy.ZSTD_COMPRESSION_LEVELS[zstdLevel]; found {
|
||||
compressionLevel = level
|
||||
} else {
|
||||
duplicacy.LOG_ERROR("STORAGE_COMPRESSION", "Invalid zstd compression level: %s", zstdLevel)
|
||||
}
|
||||
} else if context.Bool("zstd") {
|
||||
compressionLevel = duplicacy.ZSTD_COMPRESSION_LEVEL_DEFAULT
|
||||
}
|
||||
|
||||
duplicacy.ConfigStorage(storage, iterations, compressionLevel, averageChunkSize, maximumChunkSize,
|
||||
minimumChunkSize, storagePassword, otherConfig, bitCopy, context.String("key"), dataShards, parityShards)
|
||||
}
|
||||
@@ -696,7 +703,7 @@ func changePassword(context *cli.Context) {
|
||||
}
|
||||
|
||||
configPath := path.Join(duplicacy.GetDuplicacyPreferencePath(), "config")
|
||||
err = ioutil.WriteFile(configPath, description, 0600)
|
||||
err = os.WriteFile(configPath, description, 0600)
|
||||
if err != nil {
|
||||
duplicacy.LOG_ERROR("CONFIG_SAVE", "Failed to save the old config to %s: %v", configPath, err)
|
||||
return
|
||||
@@ -780,12 +787,33 @@ func backupRepository(context *cli.Context) {
|
||||
uploadRateLimit := context.Int("limit-rate")
|
||||
enumOnly := context.Bool("enum-only")
|
||||
storage.SetRateLimits(0, uploadRateLimit)
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile, preference.ExcludeByAttribute)
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password,
|
||||
&duplicacy.BackupManagerOptions{
|
||||
NobackupFile: preference.NobackupFile,
|
||||
FiltersFile: preference.FiltersFile,
|
||||
ExcludeByAttribute: preference.ExcludeByAttribute,
|
||||
ExcludeXattrs: preference.ExcludeXattrs,
|
||||
NormalizeXattrs: preference.NormalizeXattrs,
|
||||
IncludeFileFlags: preference.IncludeFileFlags,
|
||||
IncludeSpecials: preference.IncludeSpecials,
|
||||
FileFlagsMask: uint32(preference.FileFlagsMask),
|
||||
})
|
||||
duplicacy.SavePassword(*preference, "password", password)
|
||||
|
||||
backupManager.SetupSnapshotCache(preference.Name)
|
||||
backupManager.SetDryRun(dryRun)
|
||||
|
||||
zstdLevel := context.String("zstd-level")
|
||||
if zstdLevel != "" {
|
||||
if level, found := duplicacy.ZSTD_COMPRESSION_LEVELS[zstdLevel]; found {
|
||||
backupManager.SetCompressionLevel(level)
|
||||
} else {
|
||||
duplicacy.LOG_ERROR("STORAGE_COMPRESSION", "Invalid zstd compression level: %s", zstdLevel)
|
||||
}
|
||||
} else if context.Bool("zstd") {
|
||||
backupManager.SetCompressionLevel(duplicacy.ZSTD_COMPRESSION_LEVEL_DEFAULT)
|
||||
}
|
||||
|
||||
metadataChunkSize := context.Int("metadata-chunk-size")
|
||||
maximumInMemoryEntries := context.Int("max-in-memory-entries")
|
||||
backupManager.Backup(repository, quickMode, threads, context.String("t"), showStatistics, enableVSS, vssTimeout, enumOnly, metadataChunkSize, maximumInMemoryEntries)
|
||||
@@ -830,14 +858,6 @@ func restoreRepository(context *cli.Context) {
|
||||
password = duplicacy.GetPassword(*preference, "password", "Enter storage password:", false, false)
|
||||
}
|
||||
|
||||
quickMode := !context.Bool("hash")
|
||||
overwrite := context.Bool("overwrite")
|
||||
deleteMode := context.Bool("delete")
|
||||
setOwner := !context.Bool("ignore-owner")
|
||||
|
||||
showStatistics := context.Bool("stats")
|
||||
persist := context.Bool("persist")
|
||||
|
||||
var patterns []string
|
||||
for _, pattern := range context.Args() {
|
||||
|
||||
@@ -861,13 +881,38 @@ func restoreRepository(context *cli.Context) {
|
||||
duplicacy.LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns))
|
||||
|
||||
storage.SetRateLimits(context.Int("limit-rate"), 0)
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile, preference.ExcludeByAttribute)
|
||||
|
||||
excludeOwner := preference.ExcludeOwner
|
||||
// TODO: for backward compat, eventually make them all overridable?
|
||||
if context.IsSet("ignore-owner") {
|
||||
excludeOwner = context.Bool("ignore-owner")
|
||||
}
|
||||
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password,
|
||||
&duplicacy.BackupManagerOptions{
|
||||
NobackupFile: preference.NobackupFile,
|
||||
FiltersFile: preference.FiltersFile,
|
||||
ExcludeByAttribute: preference.ExcludeByAttribute,
|
||||
SetOwner: excludeOwner,
|
||||
ExcludeXattrs: preference.ExcludeXattrs,
|
||||
NormalizeXattrs: preference.NormalizeXattrs,
|
||||
IncludeSpecials: preference.IncludeSpecials,
|
||||
FileFlagsMask: uint32(preference.FileFlagsMask),
|
||||
})
|
||||
|
||||
duplicacy.SavePassword(*preference, "password", password)
|
||||
|
||||
loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false)
|
||||
|
||||
backupManager.SetupSnapshotCache(preference.Name)
|
||||
failed := backupManager.Restore(repository, revision, true, quickMode, threads, overwrite, deleteMode, setOwner, showStatistics, patterns, persist)
|
||||
failed := backupManager.Restore(repository, revision, &duplicacy.RestoreOptions{
|
||||
InPlace: true,
|
||||
QuickMode: !context.Bool("hash"),
|
||||
Overwrite: context.Bool("overwrite"),
|
||||
DeleteMode: context.Bool("delete"),
|
||||
ShowStatistics: context.Bool("stats"),
|
||||
AllowFailures: context.Bool("persist"),
|
||||
})
|
||||
if failed > 0 {
|
||||
duplicacy.LOG_ERROR("RESTORE_FAIL", "%d file(s) were not restored correctly", failed)
|
||||
return
|
||||
@@ -907,7 +952,8 @@ func listSnapshots(context *cli.Context) {
|
||||
tag := context.String("t")
|
||||
revisions := getRevisions(context)
|
||||
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", preference.ExcludeByAttribute)
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password,
|
||||
&duplicacy.BackupManagerOptions{ExcludeByAttribute: preference.ExcludeByAttribute})
|
||||
duplicacy.SavePassword(*preference, "password", password)
|
||||
|
||||
id := preference.SnapshotID
|
||||
@@ -963,7 +1009,7 @@ func checkSnapshots(context *cli.Context) {
|
||||
tag := context.String("t")
|
||||
revisions := getRevisions(context)
|
||||
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false)
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil)
|
||||
duplicacy.SavePassword(*preference, "password", password)
|
||||
|
||||
loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false)
|
||||
@@ -981,10 +1027,11 @@ func checkSnapshots(context *cli.Context) {
|
||||
checkChunks := context.Bool("chunks")
|
||||
searchFossils := context.Bool("fossils")
|
||||
resurrect := context.Bool("resurrect")
|
||||
rewrite := context.Bool("rewrite")
|
||||
persist := context.Bool("persist")
|
||||
|
||||
backupManager.SetupSnapshotCache(preference.Name)
|
||||
backupManager.SnapshotManager.CheckSnapshots(id, revisions, tag, showStatistics, showTabular, checkFiles, checkChunks, searchFossils, resurrect, threads, persist)
|
||||
backupManager.SnapshotManager.CheckSnapshots(id, revisions, tag, showStatistics, showTabular, checkFiles, checkChunks, searchFossils, resurrect, rewrite, threads, persist)
|
||||
|
||||
runScript(context, preference.Name, "post")
|
||||
}
|
||||
@@ -1022,8 +1069,7 @@ func printFile(context *cli.Context) {
|
||||
snapshotID = context.String("id")
|
||||
}
|
||||
|
||||
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false)
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil)
|
||||
duplicacy.SavePassword(*preference, "password", password)
|
||||
|
||||
loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false)
|
||||
@@ -1081,13 +1127,14 @@ func diff(context *cli.Context) {
|
||||
}
|
||||
|
||||
compareByHash := context.Bool("hash")
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false)
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil)
|
||||
duplicacy.SavePassword(*preference, "password", password)
|
||||
|
||||
loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false)
|
||||
|
||||
backupManager.SetupSnapshotCache(preference.Name)
|
||||
backupManager.SnapshotManager.Diff(repository, snapshotID, revisions, path, compareByHash, preference.NobackupFile, preference.FiltersFile, preference.ExcludeByAttribute)
|
||||
backupManager.SnapshotManager.Diff(repository, snapshotID, revisions, path, compareByHash,
|
||||
duplicacy.NewListFilesOptions(preference))
|
||||
|
||||
runScript(context, preference.Name, "post")
|
||||
}
|
||||
@@ -1126,7 +1173,7 @@ func showHistory(context *cli.Context) {
|
||||
|
||||
revisions := getRevisions(context)
|
||||
showLocalHash := context.Bool("hash")
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false)
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil)
|
||||
duplicacy.SavePassword(*preference, "password", password)
|
||||
|
||||
backupManager.SetupSnapshotCache(preference.Name)
|
||||
@@ -1189,7 +1236,7 @@ func pruneSnapshots(context *cli.Context) {
|
||||
os.Exit(ArgumentExitCode)
|
||||
}
|
||||
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false)
|
||||
backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, nil)
|
||||
duplicacy.SavePassword(*preference, "password", password)
|
||||
|
||||
backupManager.SetupSnapshotCache(preference.Name)
|
||||
@@ -1234,7 +1281,7 @@ func copySnapshots(context *cli.Context) {
|
||||
sourcePassword = duplicacy.GetPassword(*source, "password", "Enter source storage password:", false, false)
|
||||
}
|
||||
|
||||
sourceManager := duplicacy.CreateBackupManager(source.SnapshotID, sourceStorage, repository, sourcePassword, "", "", false)
|
||||
sourceManager := duplicacy.CreateBackupManager(source.SnapshotID, sourceStorage, repository, sourcePassword, nil)
|
||||
sourceManager.SetupSnapshotCache(source.Name)
|
||||
duplicacy.SavePassword(*source, "password", sourcePassword)
|
||||
|
||||
@@ -1269,7 +1316,7 @@ func copySnapshots(context *cli.Context) {
|
||||
destinationStorage.SetRateLimits(0, context.Int("upload-limit-rate"))
|
||||
|
||||
destinationManager := duplicacy.CreateBackupManager(destination.SnapshotID, destinationStorage, repository,
|
||||
destinationPassword, "", "", false)
|
||||
destinationPassword, nil)
|
||||
duplicacy.SavePassword(*destination, "password", destinationPassword)
|
||||
destinationManager.SetupSnapshotCache(destination.Name)
|
||||
|
||||
@@ -1394,7 +1441,7 @@ func benchmark(context *cli.Context) {
|
||||
if storage == nil {
|
||||
return
|
||||
}
|
||||
duplicacy.Benchmark(repository, storage, int64(fileSize) * 1024 * 1024, chunkSize * 1024 * 1024, chunkCount, uploadThreads, downloadThreads)
|
||||
duplicacy.Benchmark(repository, storage, int64(fileSize)*1024*1024, chunkSize*1024*1024, chunkCount, uploadThreads, downloadThreads)
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -1427,6 +1474,15 @@ func main() {
|
||||
Usage: "the minimum size of chunks (defaults to chunk-size/4)",
|
||||
Argument: "<size>",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "zstd-level",
|
||||
Usage: "set zstd compression level (fast, default, better, or best)",
|
||||
Argument: "<level>",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "zstd",
|
||||
Usage: "short for -zstd default",
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "iterations",
|
||||
Usage: "the number of iterations used in storage key derivation (default is 16384)",
|
||||
@@ -1494,6 +1550,15 @@ func main() {
|
||||
Name: "dry-run",
|
||||
Usage: "dry run for testing, don't backup anything. Use with -stats and -d",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "zstd-level",
|
||||
Usage: "set zstd compression level (fast, default, better, or best)",
|
||||
Argument: "<level>",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "zstd",
|
||||
Usage: "short for -zstd default",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "vss",
|
||||
Usage: "enable the Volume Shadow Copy service (Windows and macOS using APFS only)",
|
||||
@@ -1525,7 +1590,6 @@ func main() {
|
||||
Usage: "the maximum number of entries kept in memory (defaults to 1M)",
|
||||
Argument: "<number>",
|
||||
},
|
||||
|
||||
},
|
||||
Usage: "Save a snapshot of the repository to the storage",
|
||||
ArgsUsage: " ",
|
||||
@@ -1585,7 +1649,7 @@ func main() {
|
||||
cli.BoolFlag{
|
||||
Name: "persist",
|
||||
Usage: "continue processing despite chunk errors or existing files (without -overwrite), reporting any affected files",
|
||||
},
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "key-passphrase",
|
||||
Usage: "the passphrase to decrypt the RSA private key",
|
||||
@@ -1676,6 +1740,10 @@ func main() {
|
||||
Name: "resurrect",
|
||||
Usage: "turn referenced fossils back into chunks",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "rewrite",
|
||||
Usage: "rewrite chunks with recoverable corruption",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "files",
|
||||
Usage: "verify the integrity of every file",
|
||||
@@ -1933,6 +2001,15 @@ func main() {
|
||||
Usage: "the minimum size of chunks (default is chunk-size/4)",
|
||||
Argument: "<size>",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "zstd-level",
|
||||
Usage: "set zstd compression level (fast, default, better, or best)",
|
||||
Argument: "<level>",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "zstd",
|
||||
Usage: "short for -zstd default",
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "iterations",
|
||||
Usage: "the number of iterations used in storage key derivation (default is 16384)",
|
||||
@@ -2196,8 +2273,8 @@ func main() {
|
||||
Usage: "add a comment to identify the process",
|
||||
},
|
||||
cli.StringSliceFlag{
|
||||
Name: "suppress, s",
|
||||
Usage: "suppress logs with the specified id",
|
||||
Name: "suppress, s",
|
||||
Usage: "suppress logs with the specified id",
|
||||
Argument: "<id>",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
@@ -2210,7 +2287,7 @@ func main() {
|
||||
app.Name = "duplicacy"
|
||||
app.HelpName = "duplicacy"
|
||||
app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
|
||||
app.Version = "3.0.1" + " (" + GitCommit + ")"
|
||||
app.Version = "3.2.3" + " (" + GitCommit + ")"
|
||||
|
||||
// Exit with code 2 if an invalid command is provided
|
||||
app.CommandNotFound = func(context *cli.Context, command string) {
|
||||
|
||||
47
go.mod
47
go.mod
@@ -1,6 +1,6 @@
|
||||
module github.com/gilbertchen/duplicacy
|
||||
|
||||
go 1.16
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.38.0
|
||||
@@ -9,24 +9,27 @@ require (
|
||||
github.com/bkaradzic/go-lz4 v1.0.0
|
||||
github.com/gilbertchen/azure-sdk-for-go v14.1.2-0.20180323033227-8fd4663cab7c+incompatible
|
||||
github.com/gilbertchen/cli v1.2.1-0.20160223210219-1de0a1836ce9
|
||||
github.com/gilbertchen/go-dropbox v0.0.0-20221004154447-61204091e804
|
||||
github.com/gilbertchen/go-dropbox v0.0.0-20230321030224-087ef8db1916
|
||||
github.com/gilbertchen/go-ole v1.2.0
|
||||
github.com/gilbertchen/goamz v0.0.0-20170712012135-eada9f4e8cc2
|
||||
github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c
|
||||
github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8
|
||||
github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508
|
||||
github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01
|
||||
github.com/hirochachacha/go-smb2 v1.1.0
|
||||
github.com/klauspost/compress v1.16.3
|
||||
github.com/klauspost/reedsolomon v1.9.9
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
|
||||
github.com/minio/highwayhash v1.0.1
|
||||
github.com/minio/highwayhash v1.0.2
|
||||
github.com/ncw/swift/v2 v2.0.1
|
||||
github.com/pkg/sftp v1.11.0
|
||||
github.com/pkg/xattr v0.4.1
|
||||
github.com/pkg/xattr v0.4.9
|
||||
github.com/vmihailenco/msgpack v4.0.4+incompatible
|
||||
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
|
||||
golang.org/x/crypto v0.12.0
|
||||
golang.org/x/net v0.10.0
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
|
||||
golang.org/x/sys v0.11.0
|
||||
google.golang.org/api v0.21.0
|
||||
storj.io/uplink v1.9.0
|
||||
storj.io/uplink v1.12.1
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -34,35 +37,41 @@ require (
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
|
||||
github.com/dnaeon/go-vcr v1.2.0 // indirect
|
||||
github.com/flynn/noise v1.0.0 // indirect
|
||||
github.com/geoffgarside/ber v1.1.0 // indirect
|
||||
github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 // indirect
|
||||
github.com/godbus/dbus v4.1.0+incompatible // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
|
||||
github.com/golang/protobuf v1.5.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.0.5 // indirect
|
||||
github.com/jmespath/go-jmespath v0.3.0 // indirect
|
||||
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
|
||||
github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8 // indirect
|
||||
github.com/klauspost/cpuid v1.3.1 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.12 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/marstr/guid v1.1.0 // indirect
|
||||
github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/satori/go.uuid v1.2.0 // indirect
|
||||
github.com/segmentio/go-env v1.1.0 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.17 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
|
||||
github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec // indirect
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
|
||||
github.com/zeebo/blake3 v0.2.3 // indirect
|
||||
github.com/zeebo/errs v1.3.0 // indirect
|
||||
go.opencensus.io v0.22.3 // indirect
|
||||
golang.org/x/mod v0.4.2 // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
|
||||
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
|
||||
golang.org/x/text v0.3.6 // indirect
|
||||
golang.org/x/tools v0.1.1 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
golang.org/x/mod v0.10.0 // indirect
|
||||
golang.org/x/sync v0.3.0 // indirect
|
||||
golang.org/x/term v0.11.0 // indirect
|
||||
golang.org/x/text v0.12.0 // indirect
|
||||
golang.org/x/tools v0.9.1 // indirect
|
||||
google.golang.org/appengine v1.6.5 // indirect
|
||||
google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d // indirect
|
||||
google.golang.org/grpc v1.28.1 // indirect
|
||||
storj.io/common v0.0.0-20220414110316-a5cb7172d6bf // indirect
|
||||
storj.io/drpc v0.0.30 // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
storj.io/common v0.0.0-20230920095429-0ce0a575e6f8 // indirect
|
||||
storj.io/drpc v0.0.33 // indirect
|
||||
storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect
|
||||
)
|
||||
|
||||
312
go.sum
312
go.sum
@@ -1,190 +1,126 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
|
||||
cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
|
||||
dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
|
||||
dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
|
||||
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
|
||||
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
|
||||
github.com/Azure/go-autorest v10.15.5+incompatible h1:vdxx6wM1rVkKt/3niByPVjguoLWkWImOcJNvEykgBzY=
|
||||
github.com/Azure/go-autorest v10.15.5+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a h1:pv34s756C4pEXnjgPfGYgdhg/ZdajGhyOvzx8k+23nw=
|
||||
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
|
||||
github.com/aws/aws-sdk-go v1.30.7 h1:IaXfqtioP6p9SFAnNfsqdNczbR5UNbYqvcZUSsCAdTY=
|
||||
github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
|
||||
github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
|
||||
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
|
||||
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
|
||||
github.com/calebcase/tmpfile v1.0.3 h1:BZrOWZ79gJqQ3XbAQlihYZf/YCV0H4KPIdM5K5oMpJo=
|
||||
github.com/calebcase/tmpfile v1.0.3/go.mod h1:UAUc01aHeC+pudPagY/lWvt2qS9ZO5Zzof6/tIUzqeI=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ=
|
||||
github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
|
||||
github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w=
|
||||
github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/gilbertchen/azure-sdk-for-go v14.1.2-0.20180323033227-8fd4663cab7c+incompatible h1:2fZxTUw5D9uGWnYTsU/obVavn+1qTF+TsVok3U8uN2Q=
|
||||
github.com/gilbertchen/azure-sdk-for-go v14.1.2-0.20180323033227-8fd4663cab7c+incompatible/go.mod h1:qsVRCpBUm2l0eMUeI9wZ47yzra2+lv2YkGhMZpzBVUc=
|
||||
github.com/gilbertchen/cli v1.2.1-0.20160223210219-1de0a1836ce9 h1:uMgtTp4sRJ7kMQMF3xEKeFntf3XatwkLNL/byj8v97g=
|
||||
github.com/gilbertchen/cli v1.2.1-0.20160223210219-1de0a1836ce9/go.mod h1:WOnN3JdZiZwUaYtLH2DRxe5PpD43wuOIvc/Wem/39M0=
|
||||
github.com/gilbertchen/go-dropbox v0.0.0-20221004154447-61204091e804 h1:JZ0P02xoeaITbKLFAdBfiH8SNNvKGE2Y/RLdYtWoEVE=
|
||||
github.com/gilbertchen/go-dropbox v0.0.0-20221004154447-61204091e804/go.mod h1:85+2CRHC/klHy4vEM+TYtbhDo2wMjPa4JNdVzUHsDIk=
|
||||
github.com/gilbertchen/go-dropbox v0.0.0-20230321030224-087ef8db1916 h1:7VpJiGwW51MB7yJ5e27Ar/ej8Yu7WuU2SEo409qPoNs=
|
||||
github.com/gilbertchen/go-dropbox v0.0.0-20230321030224-087ef8db1916/go.mod h1:85+2CRHC/klHy4vEM+TYtbhDo2wMjPa4JNdVzUHsDIk=
|
||||
github.com/gilbertchen/go-ole v1.2.0 h1:ay65uwxo6w8UVOxN0+fuCqUXGaXxbmkGs5m4uY6e1Zw=
|
||||
github.com/gilbertchen/go-ole v1.2.0/go.mod h1:NNiozp7QxhyGmHxxNdFKIcVaINvJFTAjBJ2gYzh8fsg=
|
||||
github.com/gilbertchen/goamz v0.0.0-20170712012135-eada9f4e8cc2 h1:VDPwi3huqeJBtymgLOvPAP4S2gbSSK/UrWVwRbRAmnw=
|
||||
github.com/gilbertchen/goamz v0.0.0-20170712012135-eada9f4e8cc2/go.mod h1:AoxJeh8meXUrSWBLiq9BJvYMd9RAAGgEUU0gSkNedRY=
|
||||
github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c h1:0SR0aXvil/eQReU0olxp/j04B+Y/47fjDMotIxaAgKo=
|
||||
github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:HDsXH7AAfDsfYYX0te4zsNbnwVvZ2RtLEOCjN4y84jw=
|
||||
github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8 h1:ijgl4Y+OKCIFiCPk/Rf9tb6PrarVqitu5TynpyCmRK0=
|
||||
github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8/go.mod h1:0lQcVva56+L1PuUFXLOsJ6arJQaU0baIH8q+IegeBhg=
|
||||
github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508 h1:SqTyk5KkNXp7zTdTttIZSDcTrL5uau4K/2OpKvgBZVI=
|
||||
github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508/go.mod h1:w/pisxUZezf2XzU9Ewjphcf6q1mZtOzKPHhJiuc8cag=
|
||||
github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01 h1:LqwS9qL6SrDkp0g0iwUkETrDdtB9gTKaIbSn9imUq5o=
|
||||
github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01/go.mod h1:TMlibuxKfkdtHyltooAw7+DHqRpaXs9nxaffk00Sh1Q=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
||||
github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 h1:G1U0vew/vA/1/hBmf1XNeyIzJJbPFVv+kb+HPl6rj6c=
|
||||
github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8/go.mod h1:/Ya1YZsqLQp17bDgHdyE9/XBR1uIH1HKasTvLxcoM/A=
|
||||
github.com/godbus/dbus v4.1.0+incompatible h1:WqqLRTsQic3apZUK9qC5sGNfXthmPXzUZ7nQPrNITa4=
|
||||
github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
|
||||
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
|
||||
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20211108044417-e9b028704de0 h1:rsq1yB2xiFLDYYaYdlGBsSkwVzsCo500wMhxvW5A/bk=
|
||||
github.com/google/pprof v0.0.0-20211108044417-e9b028704de0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
|
||||
github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU=
|
||||
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
|
||||
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
|
||||
github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI=
|
||||
github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE=
|
||||
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
|
||||
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b h1:tO4MX3k5bvV0Sjv5jYrxStMTJxf1m/TW24XRyHji4aU=
|
||||
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b/go.mod h1:q7yMR8BavTz/gBNtIT/uF487LMgcuEpNGKISLAjNQes=
|
||||
github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8 h1:+A1uT26XjTsxiUUZjAAuveILWWy+Sy2TPX8OIgGvPQE=
|
||||
github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8/go.mod h1:f0ijQHcvHYAuxX6JA/JUr/Z0FVn12D9REaT/HAWVgP4=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
|
||||
github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/cpuid v1.2.4/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
|
||||
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/reedsolomon v1.9.9 h1:qCL7LZlv17xMixl55nq2/Oa1Y86nfO8EqDfv2GHND54=
|
||||
github.com/klauspost/reedsolomon v1.9.9/go.mod h1:O7yFFHiQwDR6b2t63KPUpccPtNdp5ADgh1gg4fd12wo=
|
||||
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg=
|
||||
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
|
||||
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/marstr/guid v1.1.0 h1:/M4H/1G4avsieL6BbUwCOBzulmoeKVP5ux/3mQNnbyI=
|
||||
github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
|
||||
github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
|
||||
github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
|
||||
github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
|
||||
github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8=
|
||||
github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
|
||||
github.com/minio/highwayhash v1.0.1 h1:dZ6IIu8Z14VlC0VpfKofAhCy74wu/Qb5gcn52yWoz/0=
|
||||
github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
|
||||
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
|
||||
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
|
||||
github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104 h1:ULR/QWMgcgRiZLUjSSJMU+fW+RDMstRdmnDWj9Q+AsA=
|
||||
github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104/go.mod h1:wqKykBG2QzQDJEzvRkcS8x6MiSJkF52hXZsXcjaB3ls=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
|
||||
github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0=
|
||||
github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
|
||||
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
|
||||
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||
github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
|
||||
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
|
||||
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
|
||||
github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
@@ -192,91 +128,55 @@ github.com/pkg/sftp v1.11.0 h1:4Zv0OGbpkg4yNuUtH0s8rvoYxRCNyT29NVUo6pgPmxI=
|
||||
github.com/pkg/sftp v1.11.0/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
|
||||
github.com/pkg/xattr v0.4.1 h1:dhclzL6EqOXNaPDWqoeb9tIxATfBSmjqL0b4DpSjwRw=
|
||||
github.com/pkg/xattr v0.4.1/go.mod h1:W2cGD0TBEus7MkUgv0tNZ9JutLtVO3cXu+IBRuHqnFs=
|
||||
github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
|
||||
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI=
|
||||
github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/segmentio/go-env v1.1.0 h1:AGJ7OnCx9M5NWpkYPGYELS6III/pFSnAs1GvKWStiEo=
|
||||
github.com/segmentio/go-env v1.1.0/go.mod h1:pEKO2ieHe8zF098OMaAHw21SajMuONlnI/vJNB3pB7I=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
|
||||
github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
|
||||
github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
|
||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
|
||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
|
||||
github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
|
||||
github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
|
||||
github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
|
||||
github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
|
||||
github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
|
||||
github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
|
||||
github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
|
||||
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
|
||||
github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
|
||||
github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
|
||||
github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
|
||||
github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
|
||||
github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
|
||||
github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
|
||||
github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
|
||||
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
|
||||
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
|
||||
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.4/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes=
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.17 h1:rqIuLhRUr2UtS3WNVbPY/BwvjlwKVvSOVY5p0QVocxE=
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.17/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4=
|
||||
github.com/spacemonkeygo/monotime v0.0.0-20180824235756-e3f48a95f98a/go.mod h1:ul4bvvnCOPZgq8w0nTkSmWVg/hauVpFS97Am1YM1XXo=
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.22 h1:4/g8IVItBDKLdVnqrdHZrCVPpIrwDBzl1jrV0IHQHDU=
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.22/go.mod h1:XkZYGzknZwkD0AKUnZaSXhRiVTLCkq7CWVa3IsE72gA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
|
||||
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
||||
github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks=
|
||||
github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw=
|
||||
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
|
||||
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k=
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc=
|
||||
github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
|
||||
github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/zeebo/admission/v3 v3.0.3/go.mod h1:2OWyAS5yo0Xvj2AEUosOjTUHxaY0oIIiCrXGKCYzWpo=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
|
||||
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A=
|
||||
github.com/zeebo/assert v1.3.1/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
|
||||
github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
|
||||
github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs=
|
||||
github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
|
||||
github.com/zeebo/float16 v0.1.0/go.mod h1:fssGvvXu+XS8MH57cKmyrLB/cqioYeYX/2mXCN3a5wo=
|
||||
github.com/zeebo/incenc v0.0.0-20180505221441-0d92902eec54/go.mod h1:EI8LcOBDlSL3POyqwC1eJhOYlMBMidES+613EtmmT5w=
|
||||
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
|
||||
golang.org/x/arch v0.0.0-20190909030613-46d78d1859ac/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
|
||||
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
|
||||
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838 h1:71vQrMauZZhcTVK6KdYM+rklehEEwb3E+ZhaE5jrPrE=
|
||||
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
|
||||
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -284,39 +184,28 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
|
||||
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -324,93 +213,65 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181021155630-eda9bb28ed51/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 h1:XDXtA5hveEEV8JB2l7nhMTp3t3cHp9ZpwcdjqyEWLlo=
|
||||
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
|
||||
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
|
||||
golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
|
||||
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200425043458-8463f397d07c/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
|
||||
golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.21.0 h1:zS+Q/CJJnVlXpXQVIz+lH0ZT2lBuT2ac7XD8Y/3w6hY=
|
||||
google.golang.org/api v0.21.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||
google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d h1:I7Vuu5Ejagca+VcgfBINHke3xwjCTYnIG4Q57fv0wYY=
|
||||
google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
@@ -418,42 +279,25 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k=
|
||||
google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
|
||||
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
|
||||
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
|
||||
storj.io/common v0.0.0-20220414110316-a5cb7172d6bf h1:D5xZTDOlTTQWdAWeKKm2pFLcz1sceH+f/pVAcYB9jL8=
|
||||
storj.io/common v0.0.0-20220414110316-a5cb7172d6bf/go.mod h1:LBJrpAqL4MNSrhGEwc8SJ+tIVtgfCtFEZqDy6/0j67A=
|
||||
storj.io/drpc v0.0.30 h1:jqPe4T9KEu3CDBI05A2hCMgMSHLtd/E0N0yTF9QreIE=
|
||||
storj.io/drpc v0.0.30/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg=
|
||||
storj.io/uplink v1.9.0 h1:Zg1kX1VqOQIKm0yAukteKpLuT68Be3euyNRML612ERM=
|
||||
storj.io/uplink v1.9.0/go.mod h1:f6D8306j5mnRHnPDKWCiwtPM6ukyGg77to9LaAY9l6k=
|
||||
storj.io/common v0.0.0-20230920095429-0ce0a575e6f8 h1:i+bWPhVnNL6z/TLW3vDZytB6/0bsvJM0a1GhLCxrlxQ=
|
||||
storj.io/common v0.0.0-20230920095429-0ce0a575e6f8/go.mod h1:ZmeGPzRb2sm705Nwt/WwuH3e6mliShfvvoUNy1bb9v4=
|
||||
storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI=
|
||||
storj.io/drpc v0.0.33/go.mod h1:vR804UNzhBa49NOJ6HeLjd2H3MakC1j5Gv8bsOQT6N4=
|
||||
storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c h1:or/DtG5uaZpzimL61ahlgAA+MTYn/U3txz4fe+XBFUg=
|
||||
storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c/go.mod h1:JCuc3C0gzCJHQ4J6SOx/Yjg+QTpX0D+Fvs5H46FETCk=
|
||||
storj.io/uplink v1.12.1 h1:bDc2dI6Q7EXcvPJLZuH9jIOTIf2oKxvW3xKEA+Y5EI0=
|
||||
storj.io/uplink v1.12.1/go.mod h1:1+czctHG25pMzcUp4Mds6QnoJ7LvbgYA5d1qlpFFexg=
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
crypto_rand "crypto/rand"
|
||||
"math/rand"
|
||||
)
|
||||
|
||||
func TestACDClient(t *testing.T) {
|
||||
|
||||
acdClient, err := NewACDClient("acd-token.json")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create the ACD client: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
acdClient.TestMode = true
|
||||
|
||||
rootID, _, _, err := acdClient.ListByName("", "")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to get the root node: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if rootID == "" {
|
||||
t.Errorf("No root node")
|
||||
return
|
||||
}
|
||||
|
||||
testID, _, _, err := acdClient.ListByName(rootID, "test")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list the test directory: %v", err)
|
||||
return
|
||||
}
|
||||
if testID == "" {
|
||||
testID, err = acdClient.CreateDirectory(rootID, "test")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create the test directory: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
test1ID, _, _, err := acdClient.ListByName(testID, "test1")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list the test1 directory: %v", err)
|
||||
return
|
||||
}
|
||||
if test1ID == "" {
|
||||
test1ID, err = acdClient.CreateDirectory(testID, "test1")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create the test1 directory: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
test2ID, _, _, err := acdClient.ListByName(testID, "test2")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list the test2 directory: %v", err)
|
||||
return
|
||||
}
|
||||
if test2ID == "" {
|
||||
test2ID, err = acdClient.CreateDirectory(testID, "test2")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create the test2 directory: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("test1: %s, test2: %s\n", test1ID, test2ID)
|
||||
|
||||
numberOfFiles := 20
|
||||
maxFileSize := 64 * 1024
|
||||
|
||||
for i := 0; i < numberOfFiles; i++ {
|
||||
content := make([]byte, rand.Int()%maxFileSize+1)
|
||||
_, err = crypto_rand.Read(content)
|
||||
if err != nil {
|
||||
t.Errorf("Error generating random content: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
hasher := sha256.New()
|
||||
hasher.Write(content)
|
||||
filename := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
fmt.Printf("file: %s\n", filename)
|
||||
|
||||
_, err = acdClient.UploadFile(test1ID, filename, content, 100)
|
||||
if err != nil {
|
||||
/*if e, ok := err.(ACDError); !ok || e.Status != 409 */ {
|
||||
t.Errorf("Failed to upload the file %s: %v", filename, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entries, err := acdClient.ListEntries(test1ID, true, false)
|
||||
if err != nil {
|
||||
t.Errorf("Error list randomly generated files: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
err = acdClient.MoveFile(entry.ID, test1ID, test2ID)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to move %s: %v", entry.Name, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
entries, err = acdClient.ListEntries(test2ID, true, false)
|
||||
if err != nil {
|
||||
t.Errorf("Error list randomly generated files: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
readCloser, _, err := acdClient.DownloadFile(entry.ID)
|
||||
if err != nil {
|
||||
t.Errorf("Error downloading file %s: %v", entry.Name, err)
|
||||
return
|
||||
}
|
||||
|
||||
hasher := sha256.New()
|
||||
io.Copy(hasher, readCloser)
|
||||
hash := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
if hash != entry.Name {
|
||||
t.Errorf("File %s, hash %s", entry.Name, hash)
|
||||
}
|
||||
|
||||
readCloser.Close()
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
|
||||
err = acdClient.DeleteFile(entry.ID)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to delete the file %s: %v", entry.Name, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -329,7 +329,7 @@ func (client *B2Client) AuthorizeAccount(threadIndex int) (err error, allowed bo
|
||||
if client.DownloadURL == "" {
|
||||
client.DownloadURL = output.DownloadURL
|
||||
}
|
||||
LOG_INFO("BACKBLAZE_URL", "download URL is: %s", client.DownloadURL)
|
||||
LOG_INFO("BACKBLAZE_URL", "Download URL is: %s", client.DownloadURL)
|
||||
client.IsAuthorized = true
|
||||
|
||||
client.LastAuthorizationTime = time.Now().Unix()
|
||||
@@ -396,7 +396,7 @@ type B2ListFileNamesOutput struct {
|
||||
|
||||
func (client *B2Client) ListFileNames(threadIndex int, startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
|
||||
|
||||
maxFileCount := 1000
|
||||
maxFileCount := 10_000
|
||||
if singleFile {
|
||||
if includeVersions {
|
||||
maxFileCount = 4
|
||||
@@ -584,8 +584,26 @@ func (client *B2Client) HideFile(threadIndex int, fileName string) (fileID strin
|
||||
|
||||
func (client *B2Client) DownloadFile(threadIndex int, filePath string) (io.ReadCloser, int64, error) {
|
||||
|
||||
url := client.getDownloadURL() + "/file/" + client.BucketName + "/" + B2Escape(client.StorageDir + filePath)
|
||||
if !strings.HasSuffix(filePath, ".fsl") {
|
||||
url := client.getDownloadURL() + "/file/" + client.BucketName + "/" + B2Escape(client.StorageDir + filePath)
|
||||
|
||||
readCloser, _, len, err := client.call(threadIndex, url, http.MethodGet, make(map[string]string), 0)
|
||||
return readCloser, len, err
|
||||
}
|
||||
|
||||
// We're trying to download a fossil file. We need to find the file ID of the last 'upload' of the file.
|
||||
filePath = strings.TrimSuffix(filePath, ".fsl")
|
||||
entries, err := client.ListFileNames(threadIndex, filePath, true, true)
|
||||
fileId := ""
|
||||
for _, entry := range entries {
|
||||
if entry.FileName == filePath && entry.Action == "upload" && entry.Size > 0 {
|
||||
fileId = entry.FileID
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Proceed with the b2_download_file_by_id call
|
||||
url := client.getAPIURL() + "/b2api/v1/b2_download_file_by_id?fileId=" + fileId
|
||||
readCloser, _, len, err := client.call(threadIndex, url, http.MethodGet, make(map[string]string), 0)
|
||||
return readCloser, len, err
|
||||
}
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
crypto_rand "crypto/rand"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
)
|
||||
|
||||
func createB2ClientForTest(t *testing.T) (*B2Client, string) {
|
||||
config, err := ioutil.ReadFile("test_storage.conf")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to read config file: %v", err)
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
storages := make(map[string]map[string]string)
|
||||
|
||||
err = json.Unmarshal(config, &storages)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to parse config file: %v", err)
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
b2, found := storages["b2"]
|
||||
if !found {
|
||||
t.Errorf("Failed to find b2 config")
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
return NewB2Client(b2["account"], b2["key"], "", b2["directory"], 1), b2["bucket"]
|
||||
|
||||
}
|
||||
|
||||
func TestB2Client(t *testing.T) {
|
||||
|
||||
b2Client, bucket := createB2ClientForTest(t)
|
||||
if b2Client == nil {
|
||||
return
|
||||
}
|
||||
|
||||
b2Client.TestMode = true
|
||||
|
||||
err, _ := b2Client.AuthorizeAccount(0)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to authorize the b2 account: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
err = b2Client.FindBucket(bucket)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to find bucket '%s': %v", bucket, err)
|
||||
return
|
||||
}
|
||||
|
||||
testDirectory := "b2client_test/"
|
||||
|
||||
files, err := b2Client.ListFileNames(0, testDirectory, false, false)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list files: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
err = b2Client.DeleteFile(0, file.FileName, file.FileID)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to delete file '%s': %v", file.FileName, err)
|
||||
}
|
||||
}
|
||||
|
||||
maxSize := 10000
|
||||
for i := 0; i < 20; i++ {
|
||||
size := rand.Int()%maxSize + 1
|
||||
content := make([]byte, size)
|
||||
_, err := crypto_rand.Read(content)
|
||||
if err != nil {
|
||||
t.Errorf("Error generating random content: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
hash := sha256.Sum256(content)
|
||||
name := hex.EncodeToString(hash[:])
|
||||
|
||||
err = b2Client.UploadFile(0, testDirectory+name, content, 100)
|
||||
if err != nil {
|
||||
t.Errorf("Error uploading file '%s': %v", name, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
files, err = b2Client.ListFileNames(0, testDirectory, false, false)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list files: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
|
||||
readCloser, _, err := b2Client.DownloadFile(0, file.FileName)
|
||||
if err != nil {
|
||||
t.Errorf("Error downloading file '%s': %v", file.FileName, err)
|
||||
return
|
||||
}
|
||||
|
||||
defer readCloser.Close()
|
||||
|
||||
hasher := sha256.New()
|
||||
_, err = io.Copy(hasher, readCloser)
|
||||
|
||||
hash := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
if testDirectory+hash != file.FileName {
|
||||
t.Errorf("File %s has hash %s", file.FileName, hash)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
err = b2Client.DeleteFile(0, file.FileName, file.FileID)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to delete file '%s': %v", file.FileName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,9 +7,9 @@ package duplicacy
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -21,12 +21,11 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/vmihailenco/msgpack"
|
||||
"github.com/vmihailenco/msgpack"
|
||||
)
|
||||
|
||||
// BackupManager performs the two major operations, backup and restore, and passes other operations, mostly related to
|
||||
// snapshot management, to the snapshot manager.
|
||||
|
||||
type BackupManager struct {
|
||||
snapshotID string // Unique id for each repository
|
||||
storage Storage // the storage for storing backups
|
||||
@@ -34,23 +33,56 @@ type BackupManager struct {
|
||||
SnapshotManager *SnapshotManager // the snapshot manager
|
||||
snapshotCache *FileStorage // for copies of chunks needed by snapshots
|
||||
|
||||
config *Config // contains a number of options
|
||||
|
||||
nobackupFile string // don't backup directory when this file name is found
|
||||
filtersFile string // the path to the filters file
|
||||
excludeByAttribute bool // don't backup file based on file attribute
|
||||
config *Config // contains a number of options
|
||||
options BackupManagerOptions
|
||||
|
||||
cachePath string
|
||||
}
|
||||
|
||||
type BackupManagerOptions struct {
|
||||
NobackupFile string // don't backup directory when this file name is found
|
||||
FiltersFile string // the path to the filters file
|
||||
ExcludeByAttribute bool // don't backup file based on file attribute
|
||||
SetOwner bool
|
||||
ExcludeXattrs bool
|
||||
NormalizeXattrs bool
|
||||
IncludeFileFlags bool
|
||||
IncludeSpecials bool
|
||||
FileFlagsMask uint32
|
||||
}
|
||||
|
||||
type RestoreOptions struct {
|
||||
Threads int
|
||||
Patterns []string
|
||||
InPlace bool
|
||||
QuickMode bool
|
||||
Overwrite bool
|
||||
DeleteMode bool
|
||||
ShowStatistics bool
|
||||
AllowFailures bool
|
||||
}
|
||||
|
||||
func (manager *BackupManager) SetDryRun(dryRun bool) {
|
||||
manager.config.dryRun = dryRun
|
||||
}
|
||||
|
||||
func (manager *BackupManager) SetCompressionLevel(level int) {
|
||||
manager.config.CompressionLevel = level
|
||||
}
|
||||
|
||||
func (manager *BackupManager) Config() *Config {
|
||||
return manager.config
|
||||
}
|
||||
|
||||
func (manager *BackupManager) SnapshotCache() *FileStorage {
|
||||
return manager.snapshotCache
|
||||
}
|
||||
|
||||
// CreateBackupManager creates a backup manager using the specified 'storage'. 'snapshotID' is a unique id to
|
||||
// identify snapshots created for this repository. 'top' is the top directory of the repository. 'password' is the
|
||||
// master key which can be nil if encryption is not enabled.
|
||||
func CreateBackupManager(snapshotID string, storage Storage, top string, password string, nobackupFile string, filtersFile string, excludeByAttribute bool) *BackupManager {
|
||||
func CreateBackupManager(snapshotID string, storage Storage, top string, password string,
|
||||
options *BackupManagerOptions) *BackupManager {
|
||||
|
||||
config, _, err := DownloadConfig(storage, password)
|
||||
if err != nil {
|
||||
@@ -70,13 +102,10 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor
|
||||
|
||||
SnapshotManager: snapshotManager,
|
||||
|
||||
config: config,
|
||||
|
||||
nobackupFile: nobackupFile,
|
||||
|
||||
filtersFile: filtersFile,
|
||||
|
||||
excludeByAttribute: excludeByAttribute,
|
||||
config: config,
|
||||
}
|
||||
if options != nil {
|
||||
backupManager.options = *options
|
||||
}
|
||||
|
||||
if IsDebugging() {
|
||||
@@ -127,8 +156,7 @@ func (manager *BackupManager) SetupSnapshotCache(storageName string) bool {
|
||||
func (manager *BackupManager) Backup(top string, quickMode bool, threads int, tag string,
|
||||
showStatistics bool, shadowCopy bool, shadowCopyTimeout int, enumOnly bool, metadataChunkSize int, maximumInMemoryEntries int) bool {
|
||||
|
||||
var err error
|
||||
top, err = filepath.Abs(top)
|
||||
top, err := filepath.Abs(top)
|
||||
if err != nil {
|
||||
LOG_ERROR("REPOSITORY_ERR", "Failed to obtain the absolute path of the repository: %v", err)
|
||||
return false
|
||||
@@ -138,16 +166,18 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
||||
|
||||
LOG_DEBUG("BACKUP_PARAMETERS", "top: %s, quick: %t, tag: %s", top, quickMode, tag)
|
||||
|
||||
manager.config.PrintCompressionLevel()
|
||||
|
||||
if manager.config.DataShards != 0 && manager.config.ParityShards != 0 {
|
||||
LOG_INFO("BACKUP_ERASURECODING", "Erasure coding is enabled with %d data shards and %d parity shards",
|
||||
manager.config.DataShards, manager.config.ParityShards)
|
||||
manager.config.DataShards, manager.config.ParityShards)
|
||||
}
|
||||
|
||||
if manager.config.rsaPublicKey != nil && len(manager.config.FileKey) > 0 {
|
||||
LOG_INFO("BACKUP_KEY", "RSA encryption is enabled")
|
||||
}
|
||||
|
||||
if manager.excludeByAttribute {
|
||||
if manager.options.ExcludeByAttribute {
|
||||
LOG_INFO("BACKUP_EXCLUDE", "Exclude files with no-backup attributes")
|
||||
}
|
||||
|
||||
@@ -181,7 +211,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
||||
|
||||
// If the listing operation is fast and this is an initial backup, list all chunks and
|
||||
// put them in the cache.
|
||||
if (manager.storage.IsFastListing() && remoteSnapshot.Revision == 0) {
|
||||
if manager.storage.IsFastListing() && remoteSnapshot.Revision == 0 {
|
||||
LOG_INFO("BACKUP_LIST", "Listing all chunks")
|
||||
allChunks, _ := manager.SnapshotManager.ListAllFiles(manager.storage, "chunks/")
|
||||
|
||||
@@ -216,14 +246,14 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
||||
|
||||
var totalModifiedFileSize int64 // total size of modified files
|
||||
var uploadedModifiedFileSize int64 // portions that have been uploaded (including cache hits)
|
||||
var preservedFileSize int64 // total size of unmodified files
|
||||
var preservedFileSize int64 // total size of unmodified files
|
||||
|
||||
localSnapshot := CreateEmptySnapshot(manager.snapshotID)
|
||||
localSnapshot.Revision = remoteSnapshot.Revision + 1
|
||||
|
||||
localListingChannel := make(chan *Entry)
|
||||
remoteListingChannel := make(chan *Entry)
|
||||
chunkOperator := CreateChunkOperator(manager.config, manager.storage, manager.snapshotCache, showStatistics, threads, false)
|
||||
chunkOperator := CreateChunkOperator(manager.config, manager.storage, manager.snapshotCache, showStatistics, false, threads, false)
|
||||
|
||||
var skippedDirectories []string
|
||||
var skippedFiles []string
|
||||
@@ -232,8 +262,17 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
||||
go func() {
|
||||
// List local files
|
||||
defer CatchLogException()
|
||||
localSnapshot.ListLocalFiles(shadowTop, manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute, localListingChannel, &skippedDirectories, &skippedFiles)
|
||||
} ()
|
||||
localSnapshot.ListLocalFiles(shadowTop, localListingChannel, &skippedDirectories, &skippedFiles,
|
||||
&ListFilesOptions{
|
||||
NoBackupFile: manager.options.NobackupFile,
|
||||
FiltersFile: manager.options.FiltersFile,
|
||||
ExcludeByAttribute: manager.options.ExcludeByAttribute,
|
||||
ExcludeXattrs: manager.options.ExcludeXattrs,
|
||||
NormalizeXattr: manager.options.NormalizeXattrs,
|
||||
IncludeFileFlags: manager.options.IncludeFileFlags,
|
||||
IncludeSpecials: manager.options.IncludeSpecials,
|
||||
})
|
||||
}()
|
||||
|
||||
go func() {
|
||||
// List remote files
|
||||
@@ -255,7 +294,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
||||
})
|
||||
}
|
||||
close(remoteListingChannel)
|
||||
} ()
|
||||
}()
|
||||
|
||||
// Create the local file list
|
||||
localEntryList, err := CreateEntryList(manager.snapshotID, manager.cachePath, maximumInMemoryEntries)
|
||||
@@ -269,7 +308,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
||||
var remoteEntry *Entry
|
||||
remoteListingOK := true
|
||||
for {
|
||||
localEntry := <- localListingChannel
|
||||
localEntry := <-localListingChannel
|
||||
if localEntry == nil {
|
||||
break
|
||||
}
|
||||
@@ -283,7 +322,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
||||
compareResult = localEntry.Compare(remoteEntry)
|
||||
} else {
|
||||
if remoteListingOK {
|
||||
remoteEntry, remoteListingOK = <- remoteListingChannel
|
||||
remoteEntry, remoteListingOK = <-remoteListingChannel
|
||||
}
|
||||
if !remoteListingOK {
|
||||
compareResult = -1
|
||||
@@ -298,29 +337,30 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
||||
remoteEntry = nil
|
||||
}
|
||||
|
||||
if compareResult == 0 {
|
||||
if compareResult == 0 {
|
||||
// No need to check if it is in hash mode -- in that case remote listing is nil
|
||||
if localEntry.IsSameAs(remoteEntry) && localEntry.IsFile() {
|
||||
if localEntry.Size > 0 {
|
||||
localEntry.Hash = remoteEntry.Hash
|
||||
localEntry.StartOffset = remoteEntry.StartOffset
|
||||
localEntry.EndOffset = remoteEntry.EndOffset
|
||||
delta := remoteEntry.StartChunk - len(localEntryList.PreservedChunkHashes)
|
||||
if lastPreservedChunk != remoteEntry.StartChunk {
|
||||
lastPreservedChunk = remoteEntry.StartChunk
|
||||
localEntryList.AddPreservedChunk(remoteSnapshot.ChunkHashes[lastPreservedChunk], remoteSnapshot.ChunkLengths[lastPreservedChunk])
|
||||
} else {
|
||||
delta++
|
||||
}
|
||||
|
||||
localEntry.Hash = remoteEntry.Hash
|
||||
localEntry.StartOffset = remoteEntry.StartOffset
|
||||
localEntry.EndOffset = remoteEntry.EndOffset
|
||||
delta := remoteEntry.StartChunk - len(localEntryList.PreservedChunkHashes)
|
||||
if lastPreservedChunk != remoteEntry.StartChunk {
|
||||
lastPreservedChunk = remoteEntry.StartChunk
|
||||
localEntryList.AddPreservedChunk(remoteSnapshot.ChunkHashes[lastPreservedChunk], remoteSnapshot.ChunkLengths[lastPreservedChunk])
|
||||
} else {
|
||||
delta++
|
||||
for i := remoteEntry.StartChunk + 1; i <= remoteEntry.EndChunk; i++ {
|
||||
localEntryList.AddPreservedChunk(remoteSnapshot.ChunkHashes[i], remoteSnapshot.ChunkLengths[i])
|
||||
lastPreservedChunk = i
|
||||
}
|
||||
|
||||
localEntry.StartChunk = remoteEntry.StartChunk - delta
|
||||
localEntry.EndChunk = remoteEntry.EndChunk - delta
|
||||
preservedFileSize += localEntry.Size
|
||||
}
|
||||
|
||||
for i := remoteEntry.StartChunk + 1; i <= remoteEntry.EndChunk; i++ {
|
||||
localEntryList.AddPreservedChunk(remoteSnapshot.ChunkHashes[i], remoteSnapshot.ChunkLengths[i])
|
||||
lastPreservedChunk = i
|
||||
}
|
||||
|
||||
localEntry.StartChunk = remoteEntry.StartChunk - delta
|
||||
localEntry.EndChunk = remoteEntry.EndChunk - delta
|
||||
preservedFileSize += localEntry.Size
|
||||
} else {
|
||||
totalModifiedFileSize += localEntry.Size
|
||||
if localEntry.Size > 0 {
|
||||
@@ -332,8 +372,8 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
||||
// compareResult must be < 0; the local file is new
|
||||
totalModifiedFileSize += localEntry.Size
|
||||
if localEntry.Size > 0 {
|
||||
// A size of -1 indicates this is a modified file that will be uploaded
|
||||
localEntry.Size = -1
|
||||
// A size of -1 indicates this is a modified file that will be uploaded
|
||||
localEntry.Size = -1
|
||||
}
|
||||
}
|
||||
|
||||
@@ -441,7 +481,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
||||
|
||||
_, found := chunkCache[chunkID]
|
||||
if found {
|
||||
if time.Now().Unix() - lastUploadingTime > keepUploadAlive {
|
||||
if time.Now().Unix()-lastUploadingTime > keepUploadAlive {
|
||||
LOG_INFO("UPLOAD_KEEPALIVE", "Skip chunk cache to keep connection alive")
|
||||
found = false
|
||||
}
|
||||
@@ -551,7 +591,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
||||
if showStatistics {
|
||||
|
||||
LOG_INFO("BACKUP_STATS", "Files: %d total, %s bytes; %d new, %s bytes",
|
||||
localEntryList.NumberOfEntries - int64(len(skippedFiles)),
|
||||
localEntryList.NumberOfEntries-int64(len(skippedFiles)),
|
||||
PrettyNumber(preservedFileSize+uploadedFileSize),
|
||||
len(localEntryList.ModifiedEntries), PrettyNumber(uploadedFileSize))
|
||||
|
||||
@@ -616,21 +656,34 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
||||
}
|
||||
|
||||
// Restore downloads the specified snapshot, compares it with what's on the repository, and then downloads
|
||||
// files that are different. 'base' is a directory that contains files at a different revision which can
|
||||
// serve as a local cache to avoid download chunks available locally. It is perfectly ok for 'base' to be
|
||||
// the same as 'top'. 'quickMode' will bypass files with unchanged sizes and timestamps. 'deleteMode' will
|
||||
// remove local files that don't exist in the snapshot. 'patterns' is used to include/exclude certain files.
|
||||
func (manager *BackupManager) Restore(top string, revision int, inPlace bool, quickMode bool, threads int, overwrite bool,
|
||||
deleteMode bool, setOwner bool, showStatistics bool, patterns []string, allowFailures bool) int {
|
||||
// files that are different.'QuickMode' will bypass files with unchanged sizes and timestamps. 'DeleteMode' will
|
||||
// remove local files that don't exist in the snapshot. 'Patterns' is used to include/exclude certain files.
|
||||
func (manager *BackupManager) Restore(top string, revision int, options *RestoreOptions) int {
|
||||
if options.Threads < 1 {
|
||||
options.Threads = 1
|
||||
}
|
||||
|
||||
patterns := options.Patterns
|
||||
|
||||
overwrite := options.Overwrite
|
||||
allowFailures := options.AllowFailures
|
||||
|
||||
metadataOptions := RestoreMetadataOptions{
|
||||
SetOwner: manager.options.SetOwner,
|
||||
ExcludeXattrs: manager.options.ExcludeXattrs,
|
||||
NormalizeXattrs: manager.options.NormalizeXattrs,
|
||||
IncludeFileFlags: manager.options.IncludeFileFlags,
|
||||
FileFlagsMask: manager.options.FileFlagsMask,
|
||||
}
|
||||
|
||||
startTime := time.Now().Unix()
|
||||
|
||||
LOG_DEBUG("RESTORE_PARAMETERS", "top: %s, revision: %d, in-place: %t, quick: %t, delete: %t",
|
||||
top, revision, inPlace, quickMode, deleteMode)
|
||||
top, revision, options.InPlace, options.QuickMode, options.DeleteMode)
|
||||
|
||||
if !strings.HasPrefix(GetDuplicacyPreferencePath(), top) {
|
||||
LOG_INFO("RESTORE_INPLACE", "Forcing in-place mode with a non-default preference path")
|
||||
inPlace = true
|
||||
options.InPlace = true
|
||||
}
|
||||
|
||||
if len(patterns) > 0 {
|
||||
@@ -672,14 +725,24 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
|
||||
localListingChannel := make(chan *Entry)
|
||||
remoteListingChannel := make(chan *Entry)
|
||||
chunkOperator := CreateChunkOperator(manager.config, manager.storage, manager.snapshotCache, showStatistics, threads, false)
|
||||
chunkOperator := CreateChunkOperator(manager.config, manager.storage, manager.snapshotCache, options.ShowStatistics,
|
||||
false, options.Threads, allowFailures)
|
||||
|
||||
LOG_INFO("RESTORE_INDEXING", "Indexing %s", top)
|
||||
go func() {
|
||||
// List local files
|
||||
defer CatchLogException()
|
||||
localSnapshot.ListLocalFiles(top, manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute, localListingChannel, nil, nil)
|
||||
} ()
|
||||
localSnapshot.ListLocalFiles(top, localListingChannel, nil, nil,
|
||||
&ListFilesOptions{
|
||||
NoBackupFile: manager.options.NobackupFile,
|
||||
FiltersFile: manager.options.FiltersFile,
|
||||
ExcludeByAttribute: manager.options.ExcludeByAttribute,
|
||||
ExcludeXattrs: manager.options.ExcludeXattrs,
|
||||
NormalizeXattr: manager.options.NormalizeXattrs,
|
||||
IncludeFileFlags: manager.options.IncludeFileFlags,
|
||||
IncludeSpecials: manager.options.IncludeSpecials,
|
||||
})
|
||||
}()
|
||||
|
||||
remoteSnapshot := manager.SnapshotManager.DownloadSnapshot(manager.snapshotID, revision)
|
||||
manager.SnapshotManager.DownloadSnapshotSequences(remoteSnapshot)
|
||||
@@ -691,13 +754,47 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
return true
|
||||
})
|
||||
close(remoteListingChannel)
|
||||
} ()
|
||||
}()
|
||||
|
||||
var localEntry *Entry
|
||||
localListingOK := true
|
||||
|
||||
type hardLinkEntry struct {
|
||||
entry *Entry
|
||||
willExist bool
|
||||
}
|
||||
var hardLinkTable []hardLinkEntry
|
||||
var hardLinks []*Entry
|
||||
|
||||
restoreHardLink := func(entry *Entry, fullPath string) bool {
|
||||
if entry.IsHardLinkRoot() {
|
||||
hardLinkTable[len(hardLinkTable)-1].willExist = true
|
||||
} else if entry.IsHardLinkChild() {
|
||||
i, err := entry.GetHardLinkId()
|
||||
if err != nil {
|
||||
LOG_ERROR("RESTORE_HARDLINK", "Decode error for hard link entry %s: %v", entry.Path, err)
|
||||
return false
|
||||
}
|
||||
if !hardLinkTable[i].willExist {
|
||||
hardLinkTable[i] = hardLinkEntry{entry, true}
|
||||
} else {
|
||||
sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
|
||||
if err := MakeHardlink(sourcePath, fullPath); err != nil {
|
||||
LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s: %v", fullPath, sourcePath, err)
|
||||
}
|
||||
LOG_TRACE("DOWNLOAD_DONE", "Hard linked %s to %s", entry.Path, hardLinkTable[i].entry.Path)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
for remoteEntry := range remoteListingChannel {
|
||||
|
||||
if remoteEntry.IsHardLinkRoot() {
|
||||
hardLinkTable = append(hardLinkTable, hardLinkEntry{remoteEntry, false})
|
||||
}
|
||||
|
||||
if len(patterns) > 0 && !MatchPath(remoteEntry.Path, patterns) {
|
||||
continue
|
||||
}
|
||||
@@ -707,7 +804,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
|
||||
for {
|
||||
if localEntry == nil && localListingOK {
|
||||
localEntry, localListingOK = <- localListingChannel
|
||||
localEntry, localListingOK = <-localListingChannel
|
||||
}
|
||||
if localEntry == nil {
|
||||
compareResult = 1
|
||||
@@ -723,7 +820,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
}
|
||||
|
||||
if compareResult == 0 {
|
||||
if quickMode && localEntry.IsFile() && localEntry.IsSameAs(remoteEntry) {
|
||||
if options.QuickMode && localEntry.IsFile() && localEntry.IsSameAs(remoteEntry) {
|
||||
LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", localEntry.Path)
|
||||
skippedFileSize += localEntry.Size
|
||||
skippedFileCount++
|
||||
@@ -734,27 +831,40 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
}
|
||||
|
||||
fullPath := joinPath(top, remoteEntry.Path)
|
||||
|
||||
if remoteEntry.IsLink() {
|
||||
stat, err := os.Lstat(fullPath)
|
||||
if stat != nil {
|
||||
if stat, _ := os.Lstat(fullPath); stat != nil {
|
||||
if stat.Mode()&os.ModeSymlink != 0 {
|
||||
isRegular, link, err := Readlink(fullPath)
|
||||
if err == nil && link == remoteEntry.Link && !isRegular {
|
||||
remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
|
||||
remoteEntry.RestoreMetadata(fullPath, stat, metadataOptions)
|
||||
if remoteEntry.IsHardLinkRoot() {
|
||||
hardLinkTable[len(hardLinkTable)-1].willExist = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if !overwrite {
|
||||
LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
|
||||
"File %s already exists. Please specify the -overwrite option to overwrite", remoteEntry.Path)
|
||||
continue
|
||||
}
|
||||
|
||||
os.Remove(fullPath)
|
||||
}
|
||||
|
||||
err = os.Symlink(remoteEntry.Link, fullPath)
|
||||
if err != nil {
|
||||
if restoreHardLink(remoteEntry, fullPath) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := os.Symlink(remoteEntry.Link, fullPath); err != nil {
|
||||
LOG_ERROR("RESTORE_SYMLINK", "Can't create symlink %s: %v", remoteEntry.Path, err)
|
||||
return 0
|
||||
}
|
||||
remoteEntry.RestoreMetadata(fullPath, nil, setOwner)
|
||||
remoteEntry.RestoreMetadata(fullPath, nil, metadataOptions)
|
||||
LOG_TRACE("DOWNLOAD_DONE", "Symlink %s updated", remoteEntry.Path)
|
||||
|
||||
} else if remoteEntry.IsDir() {
|
||||
|
||||
stat, err := os.Stat(fullPath)
|
||||
@@ -773,8 +883,58 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
return 0
|
||||
}
|
||||
}
|
||||
if metadataOptions.IncludeFileFlags {
|
||||
err = remoteEntry.RestoreEarlyDirFlags(fullPath, manager.options.FileFlagsMask)
|
||||
if err != nil {
|
||||
LOG_WARN("DOWNLOAD_FLAGS", "Failed to set early file flags on %s: %v", fullPath, err)
|
||||
}
|
||||
}
|
||||
directoryEntries = append(directoryEntries, remoteEntry)
|
||||
} else if remoteEntry.IsSpecial() && manager.options.IncludeSpecials {
|
||||
if stat, _ := os.Lstat(fullPath); stat != nil {
|
||||
if remoteEntry.IsSameSpecial(stat) {
|
||||
remoteEntry.RestoreMetadata(fullPath, nil, metadataOptions)
|
||||
if remoteEntry.IsHardLinkRoot() {
|
||||
hardLinkTable[len(hardLinkTable)-1].willExist = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !overwrite {
|
||||
LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
|
||||
"File %s already exists. Please specify the -overwrite option to overwrite", remoteEntry.Path)
|
||||
continue
|
||||
}
|
||||
os.Remove(fullPath)
|
||||
}
|
||||
|
||||
if restoreHardLink(remoteEntry, fullPath) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := remoteEntry.RestoreSpecial(fullPath); err != nil {
|
||||
LOG_ERROR("RESTORE_SPECIAL", "Failed to restore special file %s: %v", fullPath, err)
|
||||
return 0
|
||||
}
|
||||
remoteEntry.RestoreMetadata(fullPath, nil, metadataOptions)
|
||||
LOG_TRACE("DOWNLOAD_DONE", "Special %s %s restored", remoteEntry.Path, remoteEntry.FmtSpecial())
|
||||
|
||||
} else {
|
||||
if remoteEntry.IsHardLinkRoot() {
|
||||
hardLinkTable[len(hardLinkTable)-1].willExist = true
|
||||
} else if remoteEntry.IsHardLinkChild() {
|
||||
i, err := remoteEntry.GetHardLinkId()
|
||||
if err != nil {
|
||||
LOG_ERROR("RESTORE_HARDLINK", "Decode error for hard link entry %s: %v", remoteEntry.Path, err)
|
||||
return 0
|
||||
}
|
||||
if !hardLinkTable[i].willExist {
|
||||
hardLinkTable[i] = hardLinkEntry{remoteEntry, true}
|
||||
} else {
|
||||
hardLinks = append(hardLinks, remoteEntry)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// We can't download files here since fileEntries needs to be sorted
|
||||
fileEntries = append(fileEntries, remoteEntry)
|
||||
totalFileSize += remoteEntry.Size
|
||||
@@ -786,7 +946,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
}
|
||||
|
||||
for localListingOK {
|
||||
localEntry, localListingOK = <- localListingChannel
|
||||
localEntry, localListingOK = <-localListingChannel
|
||||
if localEntry != nil {
|
||||
extraFiles = append(extraFiles, localEntry.Path)
|
||||
}
|
||||
@@ -829,7 +989,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
fullPath := joinPath(top, file.Path)
|
||||
stat, _ := os.Stat(fullPath)
|
||||
if stat != nil {
|
||||
if quickMode {
|
||||
if options.QuickMode {
|
||||
if file.IsSameAsFileInfo(stat) {
|
||||
LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", file.Path)
|
||||
skippedFileSize += file.Size
|
||||
@@ -861,8 +1021,8 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
}
|
||||
newFile.Close()
|
||||
|
||||
file.RestoreMetadata(fullPath, nil, setOwner)
|
||||
if !showStatistics {
|
||||
file.RestoreMetadata(fullPath, nil, metadataOptions)
|
||||
if !options.ShowStatistics {
|
||||
LOG_INFO("DOWNLOAD_DONE", "Downloaded %s (0)", file.Path)
|
||||
downloadedFileSize += file.Size
|
||||
downloadedFiles = append(downloadedFiles, file)
|
||||
@@ -871,8 +1031,13 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
continue
|
||||
}
|
||||
|
||||
downloaded, err := manager.RestoreFile(chunkDownloader, chunkMaker, file, top, inPlace, overwrite, showStatistics,
|
||||
totalFileSize, downloadedFileSize, startDownloadingTime, allowFailures)
|
||||
fileFlagsMask := metadataOptions.FileFlagsMask
|
||||
if !metadataOptions.IncludeFileFlags {
|
||||
fileFlagsMask = math.MaxUint32
|
||||
}
|
||||
downloaded, err := manager.RestoreFile(chunkDownloader, chunkMaker, file, top, options.InPlace, overwrite,
|
||||
options.ShowStatistics, totalFileSize, downloadedFileSize, startDownloadingTime, allowFailures,
|
||||
fileFlagsMask)
|
||||
if err != nil {
|
||||
// RestoreFile returned an error; if allowFailures is false RestoerFile would error out and not return so here
|
||||
// we just need to show a warning
|
||||
@@ -891,10 +1056,42 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
skippedFileSize += file.Size
|
||||
skippedFileCount++
|
||||
}
|
||||
file.RestoreMetadata(fullPath, nil, setOwner)
|
||||
file.RestoreMetadata(fullPath, nil, metadataOptions)
|
||||
}
|
||||
|
||||
if deleteMode && len(patterns) == 0 {
|
||||
for _, linkEntry := range hardLinks {
|
||||
|
||||
i, _ := linkEntry.GetHardLinkId()
|
||||
sourcePath := joinPath(top, hardLinkTable[i].entry.Path)
|
||||
fullPath := joinPath(top, linkEntry.Path)
|
||||
|
||||
if stat, _ := os.Lstat(fullPath); stat != nil {
|
||||
sourceStat, _ := os.Lstat(sourcePath)
|
||||
if os.SameFile(stat, sourceStat) {
|
||||
continue
|
||||
}
|
||||
|
||||
if sourceStat == nil {
|
||||
LOG_WERROR(allowFailures, "RESTORE_HARDLINK",
|
||||
"Target %s for hard link %s is missing", sourcePath, linkEntry.Path)
|
||||
continue
|
||||
}
|
||||
if !overwrite {
|
||||
LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
|
||||
"File %s already exists. Please specify the -overwrite option to overwrite", linkEntry.Path)
|
||||
continue
|
||||
}
|
||||
os.Remove(fullPath)
|
||||
}
|
||||
|
||||
if err := MakeHardlink(sourcePath, fullPath); err != nil {
|
||||
LOG_ERROR("RESTORE_HARDLINK", "Failed to create hard link %s to %s: %v", fullPath, sourcePath, err)
|
||||
return 0
|
||||
}
|
||||
LOG_TRACE("RESTORE_HARDLINK", "Hard linked %s to %s", linkEntry.Path, hardLinkTable[i].entry.Path)
|
||||
}
|
||||
|
||||
if options.DeleteMode && len(patterns) == 0 {
|
||||
// Reverse the order to make sure directories are empty before being deleted
|
||||
for i := range extraFiles {
|
||||
file := extraFiles[len(extraFiles)-1-i]
|
||||
@@ -906,10 +1103,10 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
|
||||
for _, entry := range directoryEntries {
|
||||
dir := joinPath(top, entry.Path)
|
||||
entry.RestoreMetadata(dir, nil, setOwner)
|
||||
entry.RestoreMetadata(dir, nil, metadataOptions)
|
||||
}
|
||||
|
||||
if showStatistics {
|
||||
if options.ShowStatistics {
|
||||
for _, file := range downloadedFiles {
|
||||
LOG_INFO("DOWNLOAD_DONE", "Downloaded %s (%d)", file.Path, file.Size)
|
||||
}
|
||||
@@ -920,7 +1117,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
}
|
||||
|
||||
LOG_INFO("RESTORE_END", "Restored %s to revision %d", top, revision)
|
||||
if showStatistics {
|
||||
if options.ShowStatistics {
|
||||
LOG_INFO("RESTORE_STATS", "Files: %d total, %s bytes", len(fileEntries), PrettySize(totalFileSize))
|
||||
LOG_INFO("RESTORE_STATS", "Downloaded %d file, %s bytes, %d chunks",
|
||||
len(downloadedFiles), PrettySize(downloadedFileSize), chunkDownloader.numberOfDownloadedChunks)
|
||||
@@ -939,55 +1136,6 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
||||
return 0
|
||||
}
|
||||
|
||||
// fileEncoder encodes one file at a time to avoid loading the full json description of the entire file tree
|
||||
// in the memory
|
||||
type fileEncoder struct {
|
||||
top string
|
||||
readAttributes bool
|
||||
files []*Entry
|
||||
currentIndex int
|
||||
buffer *bytes.Buffer
|
||||
}
|
||||
|
||||
// Read reads data from the embedded buffer
|
||||
func (encoder fileEncoder) Read(data []byte) (n int, err error) {
|
||||
return encoder.buffer.Read(data)
|
||||
}
|
||||
|
||||
// NextFile switches to the next file and generates its json description in the buffer. It also takes care of
|
||||
// the ending ']' and the commas between files.
|
||||
func (encoder *fileEncoder) NextFile() (io.Reader, bool) {
|
||||
if encoder.currentIndex == len(encoder.files) {
|
||||
return nil, false
|
||||
}
|
||||
if encoder.currentIndex == len(encoder.files)-1 {
|
||||
encoder.buffer.Write([]byte("]"))
|
||||
encoder.currentIndex++
|
||||
return encoder, true
|
||||
}
|
||||
|
||||
encoder.currentIndex++
|
||||
entry := encoder.files[encoder.currentIndex]
|
||||
if encoder.readAttributes {
|
||||
entry.ReadAttributes(encoder.top)
|
||||
}
|
||||
description, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
LOG_FATAL("SNAPSHOT_ENCODE", "Failed to encode file %s: %v", encoder.files[encoder.currentIndex].Path, err)
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if encoder.readAttributes {
|
||||
entry.Attributes = nil
|
||||
}
|
||||
|
||||
if encoder.currentIndex != 0 {
|
||||
encoder.buffer.Write([]byte(","))
|
||||
}
|
||||
encoder.buffer.Write(description)
|
||||
return encoder, true
|
||||
}
|
||||
|
||||
// UploadSnapshot uploads the specified snapshot to the storage. It turns Files, ChunkHashes, and ChunkLengths into
|
||||
// sequences of chunks, and uploads these chunks, and finally the snapshot file.
|
||||
func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top string, snapshot *Snapshot,
|
||||
@@ -1039,35 +1187,63 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
|
||||
encoder := msgpack.NewEncoder(buffer)
|
||||
metadataChunkMaker := CreateMetaDataChunkMaker(manager.config, metadataChunkSize)
|
||||
|
||||
var chunkHashes []string
|
||||
var chunkHashes []string
|
||||
var chunkLengths []int
|
||||
lastChunk := -1
|
||||
|
||||
lastEndChunk := 0
|
||||
|
||||
type hardLinkEntry struct {
|
||||
entry *Entry
|
||||
startChunk int
|
||||
}
|
||||
var hardLinkTable []hardLinkEntry
|
||||
|
||||
uploadEntryInfoFunc := func(entry *Entry) error {
|
||||
if entry.IsFile() && entry.Size > 0 {
|
||||
delta := entry.StartChunk - len(chunkHashes) + 1
|
||||
if entry.StartChunk != lastChunk {
|
||||
chunkHashes = append(chunkHashes, snapshot.ChunkHashes[entry.StartChunk])
|
||||
chunkLengths = append(chunkLengths, snapshot.ChunkLengths[entry.StartChunk])
|
||||
delta--
|
||||
}
|
||||
|
||||
delta := entry.StartChunk - len(chunkHashes) + 1
|
||||
if entry.StartChunk != lastChunk {
|
||||
chunkHashes = append(chunkHashes, snapshot.ChunkHashes[entry.StartChunk])
|
||||
chunkLengths = append(chunkLengths, snapshot.ChunkLengths[entry.StartChunk])
|
||||
delta--
|
||||
}
|
||||
for i := entry.StartChunk + 1; i <= entry.EndChunk; i++ {
|
||||
chunkHashes = append(chunkHashes, snapshot.ChunkHashes[i])
|
||||
chunkLengths = append(chunkLengths, snapshot.ChunkLengths[i])
|
||||
}
|
||||
|
||||
for i := entry.StartChunk + 1; i <= entry.EndChunk; i++ {
|
||||
chunkHashes = append(chunkHashes, snapshot.ChunkHashes[i])
|
||||
chunkLengths = append(chunkLengths, snapshot.ChunkLengths[i])
|
||||
}
|
||||
lastChunk = entry.EndChunk
|
||||
entry.StartChunk -= delta
|
||||
entry.EndChunk -= delta
|
||||
|
||||
lastChunk = entry.EndChunk
|
||||
entry.StartChunk -= delta
|
||||
entry.EndChunk -= delta
|
||||
if entry.IsHardLinkRoot() {
|
||||
hardLinkTable = append(hardLinkTable, hardLinkEntry{entry, entry.StartChunk})
|
||||
}
|
||||
|
||||
if entry.IsFile() {
|
||||
delta := entry.EndChunk - entry.StartChunk
|
||||
delta = entry.EndChunk - entry.StartChunk
|
||||
entry.StartChunk -= lastEndChunk
|
||||
lastEndChunk = entry.EndChunk
|
||||
entry.EndChunk = delta
|
||||
} else if entry.IsHardLinkChild() {
|
||||
i, err := entry.GetHardLinkId()
|
||||
if err != nil {
|
||||
LOG_ERROR("SNAPSHOT_UPLOAD", "Decode error for hard link entry %s: %v", entry.Link, err)
|
||||
return err
|
||||
}
|
||||
|
||||
targetEntry := hardLinkTable[i].entry
|
||||
var startChunk, endChunk int
|
||||
|
||||
if targetEntry.IsFile() && targetEntry.Size > 0 {
|
||||
startChunk = hardLinkTable[i].startChunk - lastEndChunk
|
||||
endChunk = targetEntry.EndChunk
|
||||
lastEndChunk = hardLinkTable[i].startChunk + endChunk
|
||||
}
|
||||
entry = entry.HardLinkTo(targetEntry, startChunk, endChunk)
|
||||
|
||||
} else if entry.IsHardLinkRoot() {
|
||||
hardLinkTable = append(hardLinkTable, hardLinkEntry{entry, 0})
|
||||
}
|
||||
|
||||
buffer.Reset()
|
||||
@@ -1136,11 +1312,13 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
|
||||
// Restore downloads a file from the storage. If 'inPlace' is false, the download file is saved first to a temporary
|
||||
// file under the .duplicacy directory and then replaces the existing one. Otherwise, the existing file will be
|
||||
// overwritten directly.
|
||||
// Return: true, nil: Restored file;
|
||||
// false, nil: Skipped file;
|
||||
// false, error: Failure to restore file (only if allowFailures == true)
|
||||
// Return: true, nil: Restored file;
|
||||
//
|
||||
// false, nil: Skipped file;
|
||||
// false, error: Failure to restore file (only if allowFailures == true)
|
||||
func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chunkMaker *ChunkMaker, entry *Entry, top string, inPlace bool, overwrite bool,
|
||||
showStatistics bool, totalFileSize int64, downloadedFileSize int64, startTime int64, allowFailures bool) (bool, error) {
|
||||
showStatistics bool, totalFileSize int64, downloadedFileSize int64, startTime int64, allowFailures bool,
|
||||
fileFlagsMask uint32) (bool, error) {
|
||||
|
||||
LOG_TRACE("DOWNLOAD_START", "Downloading %s", entry.Path)
|
||||
|
||||
@@ -1187,6 +1365,10 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
|
||||
LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing: %v", fullPath, err)
|
||||
return false, nil
|
||||
}
|
||||
err = entry.RestoreEarlyFileFlags(existingFile, fileFlagsMask)
|
||||
if err != nil {
|
||||
LOG_WARN("DOWNLOAD_FLAGS", "Failed to set early file flags on %s: %v", fullPath, err)
|
||||
}
|
||||
|
||||
n := int64(1)
|
||||
// There is a go bug on Windows (https://github.com/golang/go/issues/21681) that causes Seek to fail
|
||||
@@ -1313,7 +1495,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
|
||||
// fileHash != entry.Hash, warn/error depending on -overwrite option
|
||||
if !overwrite && !isNewFile {
|
||||
LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
|
||||
"File %s already exists. Please specify the -overwrite option to overwrite", entry.Path)
|
||||
"File %s already exists. Please specify the -overwrite option to overwrite", entry.Path)
|
||||
return false, fmt.Errorf("file exists")
|
||||
}
|
||||
|
||||
@@ -1370,6 +1552,10 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
err = entry.RestoreEarlyFileFlags(existingFile, fileFlagsMask)
|
||||
if err != nil {
|
||||
LOG_WARN("DOWNLOAD_FLAGS", "Failed to set early file flags on %s: %v", fullPath, err)
|
||||
}
|
||||
|
||||
existingFile.Seek(0, 0)
|
||||
|
||||
@@ -1452,6 +1638,10 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
|
||||
LOG_ERROR("DOWNLOAD_OPEN", "Failed to open file for writing: %v", err)
|
||||
return false, nil
|
||||
}
|
||||
err = entry.RestoreEarlyFileFlags(newFile, fileFlagsMask)
|
||||
if err != nil {
|
||||
LOG_WARN("DOWNLOAD_FLAGS", "Failed to set early file flags on %s: %v", fullPath, err)
|
||||
}
|
||||
|
||||
hasher := manager.config.NewFileHasher()
|
||||
|
||||
@@ -1551,14 +1741,14 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
|
||||
func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapshotID string,
|
||||
revisionsToBeCopied []int, uploadingThreads int, downloadingThreads int) bool {
|
||||
|
||||
if !manager.config.IsCompatiableWith(otherManager.config) {
|
||||
if !manager.config.IsCompatibleWith(otherManager.config) {
|
||||
LOG_ERROR("CONFIG_INCOMPATIBLE", "Two storages are not compatible for the copy operation")
|
||||
return false
|
||||
}
|
||||
|
||||
if otherManager.config.DataShards != 0 && otherManager.config.ParityShards != 0 {
|
||||
LOG_INFO("BACKUP_ERASURECODING", "Erasure coding is enabled for the destination storage with %d data shards and %d parity shards",
|
||||
otherManager.config.DataShards, otherManager.config.ParityShards)
|
||||
otherManager.config.DataShards, otherManager.config.ParityShards)
|
||||
}
|
||||
|
||||
if otherManager.config.rsaPublicKey != nil && len(otherManager.config.FileKey) > 0 {
|
||||
@@ -1659,15 +1849,15 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
|
||||
LOG_TRACE("SNAPSHOT_COPY", "Copying snapshot %s at revision %d", snapshot.ID, snapshot.Revision)
|
||||
|
||||
for _, chunkHash := range snapshot.FileSequence {
|
||||
chunks[chunkHash] = true // The chunk is a snapshot chunk
|
||||
chunks[chunkHash] = true // The chunk is a snapshot chunk
|
||||
}
|
||||
|
||||
for _, chunkHash := range snapshot.ChunkSequence {
|
||||
chunks[chunkHash] = true // The chunk is a snapshot chunk
|
||||
chunks[chunkHash] = true // The chunk is a snapshot chunk
|
||||
}
|
||||
|
||||
for _, chunkHash := range snapshot.LengthSequence {
|
||||
chunks[chunkHash] = true // The chunk is a snapshot chunk
|
||||
chunks[chunkHash] = true // The chunk is a snapshot chunk
|
||||
}
|
||||
|
||||
description := manager.SnapshotManager.DownloadSequence(snapshot.ChunkSequence)
|
||||
@@ -1680,7 +1870,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
|
||||
|
||||
for _, chunkHash := range snapshot.ChunkHashes {
|
||||
if _, found := chunks[chunkHash]; !found {
|
||||
chunks[chunkHash] = false // The chunk is a file chunk
|
||||
chunks[chunkHash] = false // The chunk is a file chunk
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1712,16 +1902,16 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
|
||||
}
|
||||
}
|
||||
|
||||
LOG_INFO("SNAPSHOT_COPY", "Chunks to copy: %d, to skip: %d, total: %d", len(chunksToCopy), len(chunks) - len(chunksToCopy), len(chunks))
|
||||
LOG_INFO("SNAPSHOT_COPY", "Chunks to copy: %d, to skip: %d, total: %d", len(chunksToCopy), len(chunks)-len(chunksToCopy), len(chunks))
|
||||
|
||||
chunkDownloader := CreateChunkOperator(manager.config, manager.storage, nil, false, downloadingThreads, false)
|
||||
chunkDownloader := CreateChunkOperator(manager.config, manager.storage, nil, false, false, downloadingThreads, false)
|
||||
|
||||
var uploadedBytes int64
|
||||
startTime := time.Now()
|
||||
|
||||
copiedChunks := 0
|
||||
chunkUploader := CreateChunkOperator(otherManager.config, otherManager.storage, nil, false, uploadingThreads, false)
|
||||
chunkUploader.UploadCompletionFunc = func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
|
||||
chunkUploader := CreateChunkOperator(otherManager.config, otherManager.storage, nil, false, false, uploadingThreads, false)
|
||||
chunkUploader.UploadCompletionFunc = func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
|
||||
action := "Skipped"
|
||||
if !skipped {
|
||||
copiedChunks++
|
||||
@@ -1732,11 +1922,11 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
|
||||
|
||||
elapsedTime := time.Now().Sub(startTime).Seconds()
|
||||
speed := int64(float64(atomic.LoadInt64(&uploadedBytes)) / elapsedTime)
|
||||
remainingTime := int64(float64(len(chunksToCopy) - chunkIndex - 1) / float64(chunkIndex + 1) * elapsedTime)
|
||||
percentage := float64(chunkIndex + 1) / float64(len(chunksToCopy)) * 100.0
|
||||
remainingTime := int64(float64(len(chunksToCopy)-chunkIndex-1) / float64(chunkIndex+1) * elapsedTime)
|
||||
percentage := float64(chunkIndex+1) / float64(len(chunksToCopy)) * 100.0
|
||||
LOG_INFO("COPY_PROGRESS", "%s chunk %s (%d/%d) %sB/s %s %.1f%%",
|
||||
action, chunk.GetID(), chunkIndex + 1, len(chunksToCopy),
|
||||
PrettySize(speed), PrettyTime(remainingTime), percentage)
|
||||
action, chunk.GetID(), chunkIndex+1, len(chunksToCopy),
|
||||
PrettySize(speed), PrettyTime(remainingTime), percentage)
|
||||
otherManager.config.PutChunk(chunk)
|
||||
}
|
||||
|
||||
@@ -1759,7 +1949,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
|
||||
chunkDownloader.Stop()
|
||||
chunkUploader.Stop()
|
||||
|
||||
LOG_INFO("SNAPSHOT_COPY", "Copied %d new chunks and skipped %d existing chunks", copiedChunks, len(chunks) - copiedChunks)
|
||||
LOG_INFO("SNAPSHOT_COPY", "Copied %d new chunks and skipped %d existing chunks", copiedChunks, len(chunks)-copiedChunks)
|
||||
|
||||
for _, snapshot := range snapshots {
|
||||
if revisionMap[snapshot.ID][snapshot.Revision] == false {
|
||||
|
||||
@@ -176,8 +176,6 @@ func assertRestoreFailures(t *testing.T, failedFiles int, expectedFailedFiles in
|
||||
}
|
||||
|
||||
func TestBackupManager(t *testing.T) {
|
||||
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
setTestingT(t)
|
||||
SetLoggingLevel(INFO)
|
||||
|
||||
@@ -253,15 +251,23 @@ func TestBackupManager(t *testing.T) {
|
||||
time.Sleep(time.Duration(delay) * time.Second)
|
||||
|
||||
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
|
||||
backupManager := CreateBackupManager("host1", storage, testDir, password, "", "", false)
|
||||
backupManager := CreateBackupManager("host1", storage, testDir, password, nil)
|
||||
backupManager.SetupSnapshotCache("default")
|
||||
|
||||
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
|
||||
backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 1024, 1024)
|
||||
time.Sleep(time.Duration(delay) * time.Second)
|
||||
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
|
||||
failedFiles := backupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, true,
|
||||
/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
|
||||
failedFiles := backupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: nil,
|
||||
InPlace: false,
|
||||
QuickMode: false,
|
||||
Overwrite: true,
|
||||
DeleteMode: false,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: false,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 0)
|
||||
|
||||
for _, f := range []string{"file1", "file2", "dir1/file3"} {
|
||||
@@ -285,8 +291,16 @@ func TestBackupManager(t *testing.T) {
|
||||
backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false, 0, false, 1024, 1024)
|
||||
time.Sleep(time.Duration(delay) * time.Second)
|
||||
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
|
||||
failedFiles = backupManager.Restore(testDir+"/repository2", 2 /*inPlace=*/, true /*quickMode=*/, true, threads /*overwrite=*/, true,
|
||||
/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
|
||||
failedFiles = backupManager.Restore(testDir+"/repository2", 2, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: nil,
|
||||
InPlace: true,
|
||||
QuickMode: true,
|
||||
Overwrite: true,
|
||||
DeleteMode: false,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: false,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 0)
|
||||
|
||||
for _, f := range []string{"file1", "file2", "dir1/file3"} {
|
||||
@@ -314,8 +328,16 @@ func TestBackupManager(t *testing.T) {
|
||||
createRandomFile(testDir+"/repository2/dir5/file5", 100)
|
||||
|
||||
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
|
||||
failedFiles = backupManager.Restore(testDir+"/repository2", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
|
||||
/*deleteMode=*/ true /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
|
||||
failedFiles = backupManager.Restore(testDir+"/repository2", 3, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: nil,
|
||||
InPlace: true,
|
||||
QuickMode: false,
|
||||
Overwrite: true,
|
||||
DeleteMode: true,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: false,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 0)
|
||||
|
||||
for _, f := range []string{"file1", "file2", "dir1/file3"} {
|
||||
@@ -342,8 +364,16 @@ func TestBackupManager(t *testing.T) {
|
||||
os.Remove(testDir + "/repository1/file2")
|
||||
os.Remove(testDir + "/repository1/dir1/file3")
|
||||
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
|
||||
failedFiles = backupManager.Restore(testDir+"/repository1", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
|
||||
/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, []string{"+file2", "+dir1/file3", "-*"} /*allowFailures=*/, false)
|
||||
failedFiles = backupManager.Restore(testDir+"/repository1", 3, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: []string{"+file2", "+dir1/file3", "-*"},
|
||||
InPlace: true,
|
||||
QuickMode: false,
|
||||
Overwrite: true,
|
||||
DeleteMode: false,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: false,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 0)
|
||||
|
||||
for _, f := range []string{"file1", "file2", "dir1/file3"} {
|
||||
@@ -358,16 +388,17 @@ func TestBackupManager(t *testing.T) {
|
||||
if numberOfSnapshots != 3 {
|
||||
t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots)
|
||||
}
|
||||
backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1, 2, 3} /*tag*/, "",
|
||||
/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1 /*allowFailures*/, false)
|
||||
|
||||
backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1, 2, 3} /*tag*/, "" /*showStatistics*/, false,
|
||||
/*showTabular*/ false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false /*rewiret*/, false, 1 /*allowFailures*/, false)
|
||||
backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, []int{1} /*tags*/, nil /*retentions*/, nil,
|
||||
/*exhaustive*/ false /*exclusive=*/, false /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1)
|
||||
numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false)
|
||||
if numberOfSnapshots != 2 {
|
||||
t.Errorf("Expected 2 snapshots but got %d", numberOfSnapshots)
|
||||
}
|
||||
backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3} /*tag*/, "",
|
||||
/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1 /*allowFailures*/, false)
|
||||
backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3} /*tag*/, "" /*showStatistics*/, false,
|
||||
/*showTabular*/ false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false /*rewiret*/, false, 1 /*allowFailures*/, false)
|
||||
backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false, 0, false, 1024, 1024)
|
||||
backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, nil /*tags*/, nil /*retentions*/, nil,
|
||||
/*exhaustive*/ false /*exclusive=*/, true /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1)
|
||||
@@ -375,8 +406,8 @@ func TestBackupManager(t *testing.T) {
|
||||
if numberOfSnapshots != 3 {
|
||||
t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots)
|
||||
}
|
||||
backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3, 4} /*tag*/, "",
|
||||
/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1 /*allowFailures*/, false)
|
||||
backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3, 4} /*tag*/, "" /*showStatistics*/, false,
|
||||
/*showTabular*/ false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false /*rewiret*/, false, 1 /*allowFailures*/, false)
|
||||
|
||||
/*buf := make([]byte, 1<<16)
|
||||
runtime.Stack(buf, true)
|
||||
@@ -385,7 +416,7 @@ func TestBackupManager(t *testing.T) {
|
||||
|
||||
// Create file with random file with certain seed
|
||||
func createRandomFileSeeded(path string, maxSize int, seed int64) {
|
||||
rand.Seed(seed)
|
||||
r := rand.New(rand.NewSource(seed))
|
||||
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
LOG_ERROR("RANDOM_FILE", "Can't open %s for writing: %v", path, err)
|
||||
@@ -402,7 +433,7 @@ func createRandomFileSeeded(path string, maxSize int, seed int64) {
|
||||
if bytes > cap(buffer) {
|
||||
bytes = cap(buffer)
|
||||
}
|
||||
rand.Read(buffer[:bytes])
|
||||
r.Read(buffer[:bytes])
|
||||
bytes, err = file.Write(buffer[:bytes])
|
||||
if err != nil {
|
||||
LOG_ERROR("RANDOM_FILE", "Failed to write to %s: %v", path, err)
|
||||
@@ -413,7 +444,7 @@ func createRandomFileSeeded(path string, maxSize int, seed int64) {
|
||||
}
|
||||
|
||||
func corruptFile(path string, start int, length int, seed int64) {
|
||||
rand.Seed(seed)
|
||||
r := rand.New(rand.NewSource(seed))
|
||||
|
||||
file, err := os.OpenFile(path, os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
@@ -434,7 +465,7 @@ func corruptFile(path string, start int, length int, seed int64) {
|
||||
}
|
||||
|
||||
buffer := make([]byte, length)
|
||||
rand.Read(buffer)
|
||||
r.Read(buffer)
|
||||
|
||||
_, err = file.Write(buffer)
|
||||
if err != nil {
|
||||
@@ -477,9 +508,9 @@ func TestPersistRestore(t *testing.T) {
|
||||
maxFileSize := 1000000
|
||||
//maxFileSize := 200000
|
||||
|
||||
createRandomFileSeeded(testDir+"/repository1/file1", maxFileSize,1)
|
||||
createRandomFileSeeded(testDir+"/repository1/file2", maxFileSize,2)
|
||||
createRandomFileSeeded(testDir+"/repository1/dir1/file3", maxFileSize,3)
|
||||
createRandomFileSeeded(testDir+"/repository1/file1", maxFileSize, 1)
|
||||
createRandomFileSeeded(testDir+"/repository1/file2", maxFileSize, 2)
|
||||
createRandomFileSeeded(testDir+"/repository1/dir1/file3", maxFileSize, 3)
|
||||
|
||||
threads := 1
|
||||
|
||||
@@ -529,85 +560,83 @@ func TestPersistRestore(t *testing.T) {
|
||||
|
||||
// do unencrypted backup
|
||||
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
|
||||
unencBackupManager := CreateBackupManager("host1", unencStorage, testDir, "", "", "", false)
|
||||
unencBackupManager := CreateBackupManager("host1", unencStorage, testDir, "", nil)
|
||||
unencBackupManager.SetupSnapshotCache("default")
|
||||
|
||||
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
|
||||
unencBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 1024, 1024)
|
||||
time.Sleep(time.Duration(delay) * time.Second)
|
||||
|
||||
|
||||
// do encrypted backup
|
||||
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
|
||||
encBackupManager := CreateBackupManager("host1", storage, testDir, password, "", "", false)
|
||||
encBackupManager := CreateBackupManager("host1", storage, testDir, password, nil)
|
||||
encBackupManager.SetupSnapshotCache("default")
|
||||
|
||||
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
|
||||
encBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 1024, 1024)
|
||||
time.Sleep(time.Duration(delay) * time.Second)
|
||||
|
||||
|
||||
// check snapshots
|
||||
unencBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1} /*tag*/, "",
|
||||
/*showStatistics*/ true /*showTabular*/, false /*checkFiles*/, true /*checkChunks*/, false,
|
||||
/*searchFossils*/ false /*resurrect*/, false, 1 /*allowFailures*/, false)
|
||||
/*searchFossils*/ false /*resurrect*/, false /*rewiret*/, false, 1 /*allowFailures*/, false)
|
||||
|
||||
encBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1} /*tag*/, "",
|
||||
/*showStatistics*/ true /*showTabular*/, false /*checkFiles*/, true /*checkChunks*/, false,
|
||||
/*searchFossils*/ false /*resurrect*/, false, 1 /*allowFailures*/, false)
|
||||
|
||||
/*searchFossils*/ false /*resurrect*/, false /*rewiret*/, false, 1 /*allowFailures*/, false)
|
||||
|
||||
// check functions
|
||||
checkAllUncorrupted := func(cmpRepository string) {
|
||||
for _, f := range []string{"file1", "file2", "dir1/file3"} {
|
||||
if _, err := os.Stat(testDir + cmpRepository + "/" + f); os.IsNotExist(err) {
|
||||
t.Errorf("File %s does not exist", f)
|
||||
continue
|
||||
}
|
||||
|
||||
hash1 := getFileHash(testDir + "/repository1/" + f)
|
||||
hash2 := getFileHash(testDir + cmpRepository + "/" + f)
|
||||
if hash1 != hash2 {
|
||||
t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2)
|
||||
}
|
||||
}
|
||||
}
|
||||
checkMissingFile := func(cmpRepository string, expectMissing string) {
|
||||
for _, f := range []string{"file1", "file2", "dir1/file3"} {
|
||||
_, err := os.Stat(testDir + cmpRepository + "/" + f)
|
||||
if err==nil {
|
||||
if f==expectMissing {
|
||||
t.Errorf("File %s exists, expected to be missing", f)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if os.IsNotExist(err) {
|
||||
if f!=expectMissing {
|
||||
t.Errorf("File %s does not exist", f)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
hash1 := getFileHash(testDir + "/repository1/" + f)
|
||||
hash2 := getFileHash(testDir + cmpRepository + "/" + f)
|
||||
if hash1 != hash2 {
|
||||
t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2)
|
||||
}
|
||||
}
|
||||
}
|
||||
checkCorruptedFile := func(cmpRepository string, expectCorrupted string) {
|
||||
for _, f := range []string{"file1", "file2", "dir1/file3"} {
|
||||
if _, err := os.Stat(testDir + cmpRepository + "/" + f); os.IsNotExist(err) {
|
||||
t.Errorf("File %s does not exist", f)
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
hash1 := getFileHash(testDir + "/repository1/" + f)
|
||||
hash2 := getFileHash(testDir + cmpRepository + "/" + f)
|
||||
if (f==expectCorrupted) {
|
||||
if hash1 != hash2 {
|
||||
t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2)
|
||||
}
|
||||
}
|
||||
}
|
||||
checkMissingFile := func(cmpRepository string, expectMissing string) {
|
||||
for _, f := range []string{"file1", "file2", "dir1/file3"} {
|
||||
_, err := os.Stat(testDir + cmpRepository + "/" + f)
|
||||
if err == nil {
|
||||
if f == expectMissing {
|
||||
t.Errorf("File %s exists, expected to be missing", f)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if os.IsNotExist(err) {
|
||||
if f != expectMissing {
|
||||
t.Errorf("File %s does not exist", f)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
hash1 := getFileHash(testDir + "/repository1/" + f)
|
||||
hash2 := getFileHash(testDir + cmpRepository + "/" + f)
|
||||
if hash1 != hash2 {
|
||||
t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2)
|
||||
}
|
||||
}
|
||||
}
|
||||
checkCorruptedFile := func(cmpRepository string, expectCorrupted string) {
|
||||
for _, f := range []string{"file1", "file2", "dir1/file3"} {
|
||||
if _, err := os.Stat(testDir + cmpRepository + "/" + f); os.IsNotExist(err) {
|
||||
t.Errorf("File %s does not exist", f)
|
||||
continue
|
||||
}
|
||||
|
||||
hash1 := getFileHash(testDir + "/repository1/" + f)
|
||||
hash2 := getFileHash(testDir + cmpRepository + "/" + f)
|
||||
if f == expectCorrupted {
|
||||
if hash1 == hash2 {
|
||||
t.Errorf("File %s has same hashes, expected to be corrupted: %s vs %s", f, hash1, hash2)
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
if hash1 != hash2 {
|
||||
t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2)
|
||||
@@ -618,55 +647,87 @@ func TestPersistRestore(t *testing.T) {
|
||||
|
||||
// test restore all uncorrupted to repository3
|
||||
SetDuplicacyPreferencePath(testDir + "/repository3/.duplicacy")
|
||||
failedFiles := unencBackupManager.Restore(testDir+"/repository3", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false,
|
||||
/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
|
||||
failedFiles := unencBackupManager.Restore(testDir+"/repository3", 1, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: nil,
|
||||
InPlace: true,
|
||||
QuickMode: false,
|
||||
Overwrite: false,
|
||||
DeleteMode: false,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: false,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 0)
|
||||
checkAllUncorrupted("/repository3")
|
||||
|
||||
// test for corrupt files and -persist
|
||||
// corrupt a chunk
|
||||
// corrupt a chunk
|
||||
chunkToCorrupt1 := "/4d/538e5dfd2b08e782bfeb56d1360fb5d7eb9d8c4b2531cc2fca79efbaec910c"
|
||||
// this should affect file1
|
||||
// this should affect file1
|
||||
chunkToCorrupt2 := "/2b/f953a766d0196ce026ae259e76e3c186a0e4bcd3ce10f1571d17f86f0a5497"
|
||||
// this should affect dir1/file3
|
||||
|
||||
// this should affect dir1/file3
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
if i==0 {
|
||||
if i == 0 {
|
||||
// test corrupt chunks
|
||||
corruptFile(testDir+"/unenc_storage"+"/chunks"+chunkToCorrupt1, 128, 128, 4)
|
||||
corruptFile(testDir+"/enc_storage"+"/chunks"+chunkToCorrupt2, 128, 128, 4)
|
||||
} else {
|
||||
// test missing chunks
|
||||
os.Remove(testDir+"/unenc_storage"+"/chunks"+chunkToCorrupt1)
|
||||
os.Remove(testDir+"/enc_storage"+"/chunks"+chunkToCorrupt2)
|
||||
os.Remove(testDir + "/unenc_storage" + "/chunks" + chunkToCorrupt1)
|
||||
os.Remove(testDir + "/enc_storage" + "/chunks" + chunkToCorrupt2)
|
||||
}
|
||||
|
||||
|
||||
// This is to make sure that allowFailures is set to true. Note that this is not needed
|
||||
// in the production code because chunkOperator can be only recreated multiple time in tests.
|
||||
if unencBackupManager.SnapshotManager.chunkOperator != nil {
|
||||
unencBackupManager.SnapshotManager.chunkOperator.allowFailures = true
|
||||
}
|
||||
|
||||
if encBackupManager.SnapshotManager.chunkOperator != nil {
|
||||
encBackupManager.SnapshotManager.chunkOperator.allowFailures = true
|
||||
}
|
||||
|
||||
// check snapshots with --persist (allowFailures == true)
|
||||
// this would cause a panic and os.Exit from duplicacy_log if allowFailures == false
|
||||
unencBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1} /*tag*/, "",
|
||||
/*showStatistics*/ true /*showTabular*/, false /*checkFiles*/, true /*checkChunks*/, false,
|
||||
/*searchFossils*/ false /*resurrect*/, false, 1 /*allowFailures*/, true)
|
||||
/*searchFossils*/ false /*resurrect*/, false /*rewrite*/, false, 1 /*allowFailures*/, true)
|
||||
|
||||
encBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1} /*tag*/, "",
|
||||
/*showStatistics*/ true /*showTabular*/, false /*checkFiles*/, true /*checkChunks*/, false,
|
||||
/*searchFossils*/ false /*resurrect*/, false, 1 /*allowFailures*/, true)
|
||||
/*searchFossils*/ false /*resurrect*/, false /*rewrite*/, false, 1 /*allowFailures*/, true)
|
||||
|
||||
|
||||
// test restore corrupted, inPlace = true, corrupted files will have hash failures
|
||||
os.RemoveAll(testDir+"/repository2")
|
||||
os.RemoveAll(testDir + "/repository2")
|
||||
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
|
||||
failedFiles = unencBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false,
|
||||
/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
|
||||
failedFiles = unencBackupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: nil,
|
||||
InPlace: true,
|
||||
QuickMode: false,
|
||||
Overwrite: false,
|
||||
DeleteMode: false,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: true,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 1)
|
||||
|
||||
// check restore, expect file1 to be corrupted
|
||||
checkCorruptedFile("/repository2", "file1")
|
||||
|
||||
|
||||
os.RemoveAll(testDir+"/repository2")
|
||||
os.RemoveAll(testDir + "/repository2")
|
||||
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
|
||||
failedFiles = encBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false,
|
||||
/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
|
||||
failedFiles = encBackupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: nil,
|
||||
InPlace: true,
|
||||
QuickMode: false,
|
||||
Overwrite: false,
|
||||
DeleteMode: false,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: true,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 1)
|
||||
|
||||
// check restore, expect file3 to be corrupted
|
||||
@@ -674,20 +735,35 @@ func TestPersistRestore(t *testing.T) {
|
||||
|
||||
//SetLoggingLevel(DEBUG)
|
||||
// test restore corrupted, inPlace = false, corrupted files will be missing
|
||||
os.RemoveAll(testDir+"/repository2")
|
||||
os.RemoveAll(testDir + "/repository2")
|
||||
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
|
||||
failedFiles = unencBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, false,
|
||||
/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
|
||||
failedFiles = unencBackupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: nil,
|
||||
InPlace: false,
|
||||
QuickMode: false,
|
||||
Overwrite: false,
|
||||
DeleteMode: false,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: true,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 1)
|
||||
|
||||
// check restore, expect file1 to be corrupted
|
||||
checkMissingFile("/repository2", "file1")
|
||||
|
||||
|
||||
os.RemoveAll(testDir+"/repository2")
|
||||
os.RemoveAll(testDir + "/repository2")
|
||||
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
|
||||
failedFiles = encBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, false,
|
||||
/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
|
||||
failedFiles = encBackupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: nil,
|
||||
InPlace: false,
|
||||
QuickMode: false,
|
||||
Overwrite: false,
|
||||
DeleteMode: false,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: true,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 1)
|
||||
|
||||
// check restore, expect file3 to be corrupted
|
||||
@@ -696,28 +772,60 @@ func TestPersistRestore(t *testing.T) {
|
||||
// test restore corrupted files from different backups, inPlace = true
|
||||
// with overwrite=true, corrupted file1 from unenc will be restored correctly from enc
|
||||
// the latter will not touch the existing file3 with correct hash
|
||||
os.RemoveAll(testDir+"/repository2")
|
||||
failedFiles = unencBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false,
|
||||
/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
|
||||
os.RemoveAll(testDir + "/repository2")
|
||||
failedFiles = unencBackupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: nil,
|
||||
InPlace: true,
|
||||
QuickMode: false,
|
||||
Overwrite: false,
|
||||
DeleteMode: false,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: true,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 1)
|
||||
|
||||
failedFiles = encBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
|
||||
/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
|
||||
failedFiles = encBackupManager.Restore(testDir+"/repository2", 1, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: nil,
|
||||
InPlace: true,
|
||||
QuickMode: false,
|
||||
Overwrite: true,
|
||||
DeleteMode: false,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: true,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 0)
|
||||
checkAllUncorrupted("/repository2")
|
||||
|
||||
// restore to repository3, with overwrite and allowFailures (true/false), quickMode = false (use hashes)
|
||||
// should always succeed as uncorrupted files already exist with correct hash, so these will be ignored
|
||||
SetDuplicacyPreferencePath(testDir + "/repository3/.duplicacy")
|
||||
failedFiles = unencBackupManager.Restore(testDir+"/repository3", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
|
||||
/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
|
||||
failedFiles = unencBackupManager.Restore(testDir+"/repository3", 1, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: nil,
|
||||
InPlace: true,
|
||||
QuickMode: false,
|
||||
Overwrite: true,
|
||||
DeleteMode: false,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: false,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 0)
|
||||
checkAllUncorrupted("/repository3")
|
||||
|
||||
failedFiles = unencBackupManager.Restore(testDir+"/repository3", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
|
||||
/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
|
||||
failedFiles = unencBackupManager.Restore(testDir+"/repository3", 1, &RestoreOptions{
|
||||
Threads: threads,
|
||||
Patterns: nil,
|
||||
InPlace: true,
|
||||
QuickMode: false,
|
||||
Overwrite: true,
|
||||
DeleteMode: false,
|
||||
ShowStatistics: false,
|
||||
AllowFailures: true,
|
||||
})
|
||||
assertRestoreFailures(t, failedFiles, 0)
|
||||
checkAllUncorrupted("/repository3")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,13 @@ import (
|
||||
"github.com/bkaradzic/go-lz4"
|
||||
"github.com/minio/highwayhash"
|
||||
"github.com/klauspost/reedsolomon"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
|
||||
// This is a fork of github.com/minio/highwayhash at 1.0.1 that computes incorrect hash on
|
||||
// arm64 machines. We need this fork to be able to read the chunks created by Duplicacy
|
||||
// CLI 3.0.1 which unfortunately relies on incorrect hashes to determine if each shard is valid.
|
||||
wronghighwayhash "github.com/gilbertchen/highwayhash"
|
||||
|
||||
)
|
||||
|
||||
// A chunk needs to acquire a new buffer and return the old one for every encrypt/decrypt operation, therefore
|
||||
@@ -261,6 +268,38 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string, isMetada
|
||||
deflater, _ := zlib.NewWriterLevel(encryptedBuffer, chunk.config.CompressionLevel)
|
||||
deflater.Write(chunk.buffer.Bytes())
|
||||
deflater.Close()
|
||||
} else if chunk.config.CompressionLevel >= ZSTD_COMPRESSION_LEVEL_FASTEST && chunk.config.CompressionLevel <= ZSTD_COMPRESSION_LEVEL_BEST {
|
||||
encryptedBuffer.Write([]byte("ZSTD"))
|
||||
|
||||
compressionLevel := zstd.SpeedDefault
|
||||
if chunk.config.CompressionLevel == ZSTD_COMPRESSION_LEVEL_FASTEST {
|
||||
compressionLevel = zstd.SpeedFastest
|
||||
} else if chunk.config.CompressionLevel == ZSTD_COMPRESSION_LEVEL_BETTER {
|
||||
compressionLevel = zstd.SpeedBetterCompression
|
||||
} else if chunk.config.CompressionLevel == ZSTD_COMPRESSION_LEVEL_BEST {
|
||||
compressionLevel = zstd.SpeedBestCompression
|
||||
}
|
||||
|
||||
deflater, err := zstd.NewWriter(encryptedBuffer, zstd.WithEncoderLevel(compressionLevel))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Make sure we have enough space in encryptedBuffer
|
||||
availableLength := encryptedBuffer.Cap() - len(encryptedBuffer.Bytes())
|
||||
maximumLength := deflater.MaxEncodedSize(chunk.buffer.Len())
|
||||
if availableLength < maximumLength {
|
||||
encryptedBuffer.Grow(maximumLength - availableLength)
|
||||
}
|
||||
_, err = deflater.Write(chunk.buffer.Bytes())
|
||||
if err != nil {
|
||||
return fmt.Errorf("ZSTD compression error: %v", err)
|
||||
}
|
||||
|
||||
err = deflater.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("ZSTD compression error: %v", err)
|
||||
}
|
||||
} else if chunk.config.CompressionLevel == DEFAULT_COMPRESSION_LEVEL {
|
||||
encryptedBuffer.Write([]byte("LZ4 "))
|
||||
// Make sure we have enough space in encryptedBuffer
|
||||
@@ -355,7 +394,6 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string, isMetada
|
||||
chunk.buffer.Write(header)
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// This is to ensure compatibility with Vertical Backup, which still uses HMAC-SHA256 (instead of HMAC-BLAKE2) to
|
||||
@@ -371,8 +409,9 @@ func init() {
|
||||
|
||||
// Decrypt decrypts the encrypted data stored in the chunk buffer. If derivationKey is not nil, the actual
|
||||
// encryption key will be HMAC-SHA256(encryptionKey, derivationKey).
|
||||
func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err error) {
|
||||
func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err error, rewriteNeeded bool) {
|
||||
|
||||
rewriteNeeded = false
|
||||
var offset int
|
||||
|
||||
encryptedBuffer := AllocateChunkBuffer()
|
||||
@@ -388,13 +427,13 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
|
||||
|
||||
// The chunk was encoded with erasure coding
|
||||
if len(encryptedBuffer.Bytes()) < bannerLength + 14 {
|
||||
return fmt.Errorf("Erasure coding header truncated (%d bytes)", len(encryptedBuffer.Bytes()))
|
||||
return fmt.Errorf("Erasure coding header truncated (%d bytes)", len(encryptedBuffer.Bytes())), false
|
||||
}
|
||||
// Check the header checksum
|
||||
header := encryptedBuffer.Bytes()[bannerLength: bannerLength + 14]
|
||||
if header[12] != header[0] ^ header[2] ^ header[4] ^ header[6] ^ header[8] ^ header[10] ||
|
||||
header[13] != header[1] ^ header[3] ^ header[5] ^ header[7] ^ header[9] ^ header[11] {
|
||||
return fmt.Errorf("Erasure coding header corrupted (%x)", header)
|
||||
return fmt.Errorf("Erasure coding header corrupted (%x)", header), false
|
||||
}
|
||||
|
||||
// Read the parameters
|
||||
@@ -414,7 +453,7 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
|
||||
} else if len(encryptedBuffer.Bytes()) > minimumLength {
|
||||
LOG_WARN("CHUNK_ERASURECODE", "Chunk is truncated (%d out of %d bytes)", len(encryptedBuffer.Bytes()), expectedLength)
|
||||
} else {
|
||||
return fmt.Errorf("Not enough chunk data for recovery; chunk size: %d bytes, data size: %d, parity: %d/%d", chunkSize, len(encryptedBuffer.Bytes()), dataShards, parityShards)
|
||||
return fmt.Errorf("Not enough chunk data for recovery; chunk size: %d bytes, data size: %d, parity: %d/%d", chunkSize, len(encryptedBuffer.Bytes()), dataShards, parityShards), false
|
||||
}
|
||||
|
||||
// Where the hashes start
|
||||
@@ -426,6 +465,8 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
|
||||
recoveryNeeded := false
|
||||
hashKey := make([]byte, 32)
|
||||
availableShards := 0
|
||||
wrongHashDetected := false
|
||||
|
||||
for i := 0; i < dataShards + parityShards; i++ {
|
||||
start := dataOffset + i * shardSize
|
||||
if start + shardSize > len(encryptedBuffer.Bytes()) {
|
||||
@@ -435,15 +476,34 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
|
||||
// Now verify the hash
|
||||
hasher, err := highwayhash.New(hashKey)
|
||||
if err != nil {
|
||||
return err
|
||||
return err, false
|
||||
}
|
||||
_, err = hasher.Write(encryptedBuffer.Bytes()[start: start + shardSize])
|
||||
if err != nil {
|
||||
return err
|
||||
return err, false
|
||||
}
|
||||
if bytes.Compare(hasher.Sum(nil), encryptedBuffer.Bytes()[hashOffset + i * 32: hashOffset + (i + 1) * 32]) != 0 {
|
||||
|
||||
matched := bytes.Compare(hasher.Sum(nil), encryptedBuffer.Bytes()[hashOffset + i * 32: hashOffset + (i + 1) * 32]) == 0
|
||||
|
||||
if !matched && runtime.GOARCH == "arm64" {
|
||||
hasher, err := wronghighwayhash.New(hashKey)
|
||||
if err == nil {
|
||||
_, err = hasher.Write(encryptedBuffer.Bytes()[start: start + shardSize])
|
||||
if err == nil {
|
||||
matched = bytes.Compare(hasher.Sum(nil), encryptedBuffer.Bytes()[hashOffset + i * 32: hashOffset + (i + 1) * 32]) == 0
|
||||
if matched && !wrongHashDetected {
|
||||
LOG_WARN("CHUNK_ERASURECODE", "Hash for shard %d was calculated with a wrong version of highwayhash", i)
|
||||
wrongHashDetected = true
|
||||
rewriteNeeded = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !matched {
|
||||
if i < dataShards {
|
||||
recoveryNeeded = true
|
||||
rewriteNeeded = true
|
||||
}
|
||||
} else {
|
||||
// The shard is good
|
||||
@@ -463,7 +523,7 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
|
||||
encryptedBuffer.Read(encryptedBuffer.Bytes()[:dataOffset])
|
||||
} else {
|
||||
if availableShards < dataShards {
|
||||
return fmt.Errorf("Not enough chunk data for recover; only %d out of %d shards are complete", availableShards, dataShards + parityShards)
|
||||
return fmt.Errorf("Not enough chunk data for recover; only %d out of %d shards are complete", availableShards, dataShards + parityShards), false
|
||||
}
|
||||
|
||||
// Show the validity of shards using a string of * and -
|
||||
@@ -479,11 +539,11 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
|
||||
LOG_WARN("CHUNK_ERASURECODE", "Recovering a %d byte chunk from %d byte shards: %s", chunkSize, shardSize, slots)
|
||||
encoder, err := reedsolomon.New(dataShards, parityShards)
|
||||
if err != nil {
|
||||
return err
|
||||
return err, false
|
||||
}
|
||||
err = encoder.Reconstruct(data)
|
||||
if err != nil {
|
||||
return err
|
||||
return err, false
|
||||
}
|
||||
LOG_DEBUG("CHUNK_ERASURECODE", "Chunk data successfully recovered")
|
||||
buffer := AllocateChunkBuffer()
|
||||
@@ -516,28 +576,28 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
|
||||
}
|
||||
|
||||
if len(encryptedBuffer.Bytes()) < bannerLength + 12 {
|
||||
return fmt.Errorf("No enough encrypted data (%d bytes) provided", len(encryptedBuffer.Bytes()))
|
||||
return fmt.Errorf("No enough encrypted data (%d bytes) provided", len(encryptedBuffer.Bytes())), false
|
||||
}
|
||||
|
||||
if string(encryptedBuffer.Bytes()[:bannerLength-1]) != ENCRYPTION_BANNER[:bannerLength-1] {
|
||||
return fmt.Errorf("The storage doesn't seem to be encrypted")
|
||||
return fmt.Errorf("The storage doesn't seem to be encrypted"), false
|
||||
}
|
||||
|
||||
encryptionVersion := encryptedBuffer.Bytes()[bannerLength-1]
|
||||
if encryptionVersion != 0 && encryptionVersion != ENCRYPTION_VERSION_RSA {
|
||||
return fmt.Errorf("Unsupported encryption version %d", encryptionVersion)
|
||||
return fmt.Errorf("Unsupported encryption version %d", encryptionVersion), false
|
||||
}
|
||||
|
||||
if encryptionVersion == ENCRYPTION_VERSION_RSA {
|
||||
if chunk.config.rsaPrivateKey == nil {
|
||||
LOG_ERROR("CHUNK_DECRYPT", "An RSA private key is required to decrypt the chunk")
|
||||
return fmt.Errorf("An RSA private key is required to decrypt the chunk")
|
||||
return fmt.Errorf("An RSA private key is required to decrypt the chunk"), false
|
||||
}
|
||||
|
||||
encryptedKeyLength := binary.LittleEndian.Uint16(encryptedBuffer.Bytes()[bannerLength:bannerLength+2])
|
||||
|
||||
if len(encryptedBuffer.Bytes()) < bannerLength + 14 + int(encryptedKeyLength) {
|
||||
return fmt.Errorf("No enough encrypted data (%d bytes) provided", len(encryptedBuffer.Bytes()))
|
||||
return fmt.Errorf("No enough encrypted data (%d bytes) provided", len(encryptedBuffer.Bytes())), false
|
||||
}
|
||||
|
||||
encryptedKey := encryptedBuffer.Bytes()[bannerLength + 2:bannerLength + 2 + int(encryptedKeyLength)]
|
||||
@@ -545,19 +605,19 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
|
||||
|
||||
decryptedKey, err := rsa.DecryptOAEP(sha256.New(), rand.Reader, chunk.config.rsaPrivateKey, encryptedKey, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
return err, false
|
||||
}
|
||||
key = decryptedKey
|
||||
}
|
||||
|
||||
aesBlock, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return err
|
||||
return err, false
|
||||
}
|
||||
|
||||
gcm, err := cipher.NewGCM(aesBlock)
|
||||
if err != nil {
|
||||
return err
|
||||
return err, false
|
||||
}
|
||||
|
||||
offset = bannerLength + gcm.NonceSize()
|
||||
@@ -567,7 +627,7 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
|
||||
encryptedBuffer.Bytes()[offset:], nil)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
return err, false
|
||||
}
|
||||
|
||||
paddingLength := int(decryptedBytes[len(decryptedBytes)-1])
|
||||
@@ -575,14 +635,14 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
|
||||
paddingLength = 256
|
||||
}
|
||||
if len(decryptedBytes) <= paddingLength {
|
||||
return fmt.Errorf("Incorrect padding length %d out of %d bytes", paddingLength, len(decryptedBytes))
|
||||
return fmt.Errorf("Incorrect padding length %d out of %d bytes", paddingLength, len(decryptedBytes)), false
|
||||
}
|
||||
|
||||
for i := 0; i < paddingLength; i++ {
|
||||
padding := decryptedBytes[len(decryptedBytes)-1-i]
|
||||
if padding != byte(paddingLength) {
|
||||
return fmt.Errorf("Incorrect padding of length %d: %x", paddingLength,
|
||||
decryptedBytes[len(decryptedBytes)-paddingLength:])
|
||||
decryptedBytes[len(decryptedBytes)-paddingLength:]), false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -596,18 +656,36 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
|
||||
chunk.buffer.Reset()
|
||||
decompressed, err := lz4.Decode(chunk.buffer.Bytes(), encryptedBuffer.Bytes()[4:])
|
||||
if err != nil {
|
||||
return err
|
||||
return err, false
|
||||
}
|
||||
|
||||
chunk.buffer.Write(decompressed)
|
||||
chunk.hasher = chunk.config.NewKeyedHasher(chunk.config.HashKey)
|
||||
chunk.hasher.Write(decompressed)
|
||||
chunk.hash = nil
|
||||
return nil
|
||||
return nil, rewriteNeeded
|
||||
}
|
||||
|
||||
if len(compressed) > 4 && string(compressed[:4]) == "ZSTD" {
|
||||
chunk.buffer.Reset()
|
||||
chunk.hasher = chunk.config.NewKeyedHasher(chunk.config.HashKey)
|
||||
chunk.hash = nil
|
||||
|
||||
encryptedBuffer.Read(encryptedBuffer.Bytes()[:4])
|
||||
inflater, err := zstd.NewReader(encryptedBuffer)
|
||||
if err != nil {
|
||||
return err, false
|
||||
}
|
||||
defer inflater.Close()
|
||||
if _, err = io.Copy(chunk, inflater); err != nil {
|
||||
return err, false
|
||||
}
|
||||
return nil, rewriteNeeded
|
||||
}
|
||||
|
||||
inflater, err := zlib.NewReader(encryptedBuffer)
|
||||
if err != nil {
|
||||
return err
|
||||
return err, false
|
||||
}
|
||||
|
||||
defer inflater.Close()
|
||||
@@ -617,9 +695,9 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
|
||||
chunk.hash = nil
|
||||
|
||||
if _, err = io.Copy(chunk, inflater); err != nil {
|
||||
return err
|
||||
return err, false
|
||||
}
|
||||
|
||||
return nil
|
||||
return nil, rewriteNeeded
|
||||
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ func TestErasureCoding(t *testing.T) {
|
||||
|
||||
chunk.Reset(false)
|
||||
chunk.Write(encryptedData)
|
||||
err = chunk.Decrypt([]byte(""), "")
|
||||
err, _ = chunk.Decrypt([]byte(""), "")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to decrypt the data: %v", err)
|
||||
return
|
||||
@@ -110,7 +110,7 @@ func TestChunkBasic(t *testing.T) {
|
||||
|
||||
chunk.Reset(false)
|
||||
chunk.Write(encryptedData)
|
||||
err = chunk.Decrypt(key, "")
|
||||
err, _ = chunk.Decrypt(key, "")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to decrypt the data: %v", err)
|
||||
continue
|
||||
|
||||
@@ -57,11 +57,14 @@ type ChunkOperator struct {
|
||||
allowFailures bool // Whether to fail on download error, or continue
|
||||
NumberOfFailedChunks int64 // The number of chunks that can't be downloaded
|
||||
|
||||
rewriteChunks bool // Whether to rewrite corrupted chunks when erasure coding is enabled
|
||||
|
||||
UploadCompletionFunc func(chunk *Chunk, chunkIndex int, inCache bool, chunkSize int, uploadSize int)
|
||||
}
|
||||
|
||||
// CreateChunkOperator creates a new ChunkOperator.
|
||||
func CreateChunkOperator(config *Config, storage Storage, snapshotCache *FileStorage, showStatistics bool, threads int, allowFailures bool) *ChunkOperator {
|
||||
func CreateChunkOperator(config *Config, storage Storage, snapshotCache *FileStorage, showStatistics bool, rewriteChunks bool, threads int,
|
||||
allowFailures bool) *ChunkOperator {
|
||||
|
||||
operator := &ChunkOperator{
|
||||
config: config,
|
||||
@@ -76,6 +79,7 @@ func CreateChunkOperator(config *Config, storage Storage, snapshotCache *FileSto
|
||||
collectionLock: &sync.Mutex{},
|
||||
startTime: time.Now().Unix(),
|
||||
allowFailures: allowFailures,
|
||||
rewriteChunks: rewriteChunks,
|
||||
}
|
||||
|
||||
// Start the operator goroutines
|
||||
@@ -331,24 +335,34 @@ func (operator *ChunkOperator) DownloadChunk(threadIndex int, task ChunkTask) {
|
||||
|
||||
atomic.AddInt64(&operator.NumberOfFailedChunks, 1)
|
||||
if operator.allowFailures {
|
||||
chunk.isBroken = true
|
||||
task.completionFunc(chunk, task.chunkIndex)
|
||||
}
|
||||
}
|
||||
|
||||
chunkPath := ""
|
||||
fossilPath := ""
|
||||
filePath := ""
|
||||
|
||||
const MaxDownloadAttempts = 3
|
||||
for downloadAttempt := 0; ; downloadAttempt++ {
|
||||
|
||||
exist := false
|
||||
var err error
|
||||
|
||||
// Find the chunk by ID first.
|
||||
chunkPath, exist, _, err := operator.storage.FindChunk(threadIndex, chunkID, false)
|
||||
chunkPath, exist, _, err = operator.storage.FindChunk(threadIndex, chunkID, false)
|
||||
if err != nil {
|
||||
completeFailedChunk()
|
||||
LOG_WERROR(operator.allowFailures, "DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
|
||||
return
|
||||
}
|
||||
|
||||
if !exist {
|
||||
if exist {
|
||||
filePath = chunkPath
|
||||
} else {
|
||||
// No chunk is found. Have to find it in the fossil pool again.
|
||||
fossilPath, exist, _, err := operator.storage.FindChunk(threadIndex, chunkID, true)
|
||||
fossilPath, exist, _, err = operator.storage.FindChunk(threadIndex, chunkID, true)
|
||||
if err != nil {
|
||||
completeFailedChunk()
|
||||
LOG_WERROR(operator.allowFailures, "DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
|
||||
@@ -383,20 +397,11 @@ func (operator *ChunkOperator) DownloadChunk(threadIndex int, task ChunkTask) {
|
||||
return
|
||||
}
|
||||
|
||||
// We can't download the fossil directly. We have to turn it back into a regular chunk and try
|
||||
// downloading again.
|
||||
err = operator.storage.MoveFile(threadIndex, fossilPath, chunkPath)
|
||||
if err != nil {
|
||||
completeFailedChunk()
|
||||
LOG_WERROR(operator.allowFailures, "DOWNLOAD_CHUNK", "Failed to resurrect chunk %s: %v", chunkID, err)
|
||||
return
|
||||
}
|
||||
|
||||
LOG_WARN("DOWNLOAD_RESURRECT", "Fossil %s has been resurrected", chunkID)
|
||||
continue
|
||||
filePath = fossilPath
|
||||
LOG_WARN("DOWNLOAD_FOSSIL", "Chunk %s is a fossil", chunkID)
|
||||
}
|
||||
|
||||
err = operator.storage.DownloadFile(threadIndex, chunkPath, chunk)
|
||||
err = operator.storage.DownloadFile(threadIndex, filePath, chunk)
|
||||
if err != nil {
|
||||
_, isHubic := operator.storage.(*HubicStorage)
|
||||
// Retry on EOF or if it is a Hubic backend as it may return 404 even when the chunk exists
|
||||
@@ -412,7 +417,8 @@ func (operator *ChunkOperator) DownloadChunk(threadIndex int, task ChunkTask) {
|
||||
}
|
||||
}
|
||||
|
||||
err = chunk.Decrypt(operator.config.ChunkKey, task.chunkHash)
|
||||
rewriteNeeded := false
|
||||
err, rewriteNeeded = chunk.Decrypt(operator.config.ChunkKey, task.chunkHash)
|
||||
if err != nil {
|
||||
if downloadAttempt < MaxDownloadAttempts {
|
||||
LOG_WARN("DOWNLOAD_RETRY", "Failed to decrypt the chunk %s: %v; retrying", chunkID, err)
|
||||
@@ -440,6 +446,38 @@ func (operator *ChunkOperator) DownloadChunk(threadIndex int, task ChunkTask) {
|
||||
}
|
||||
}
|
||||
|
||||
if rewriteNeeded && operator.rewriteChunks {
|
||||
|
||||
if filePath != fossilPath {
|
||||
fossilPath = filePath + ".fsl"
|
||||
err := operator.storage.MoveFile(threadIndex, chunkPath, fossilPath)
|
||||
if err != nil {
|
||||
LOG_WARN("CHUNK_REWRITE", "Failed to fossilize the chunk %s: %v", task.chunkID, err)
|
||||
} else {
|
||||
LOG_TRACE("CHUNK_REWRITE", "The existing chunk %s has been marked as a fossil for rewrite", task.chunkID)
|
||||
operator.collectionLock.Lock()
|
||||
operator.fossils = append(operator.fossils, fossilPath)
|
||||
operator.collectionLock.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
newChunk := operator.config.GetChunk()
|
||||
newChunk.Reset(true)
|
||||
newChunk.Write(chunk.GetBytes())
|
||||
// Encrypt the chunk only after we know that it must be uploaded.
|
||||
err = newChunk.Encrypt(operator.config.ChunkKey, chunk.GetHash(), task.isMetadata)
|
||||
if err == nil {
|
||||
// Re-upload the chunk
|
||||
err = operator.storage.UploadFile(threadIndex, chunkPath, newChunk.GetBytes())
|
||||
if err != nil {
|
||||
LOG_WARN("CHUNK_REWRITE", "Failed to re-upload the chunk %s: %v", chunkID, err)
|
||||
} else {
|
||||
LOG_INFO("CHUNK_REWRITE", "The chunk %s has been re-uploaded", chunkID)
|
||||
}
|
||||
}
|
||||
operator.config.PutChunk(newChunk)
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ func TestChunkOperator(t *testing.T) {
|
||||
totalFileSize += chunk.GetLength()
|
||||
}
|
||||
|
||||
chunkOperator := CreateChunkOperator(config, storage, nil, false, *testThreads, false)
|
||||
chunkOperator := CreateChunkOperator(config, storage, nil, false, false, *testThreads, false)
|
||||
chunkOperator.UploadCompletionFunc = func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
|
||||
t.Logf("Chunk %s size %d (%d/%d) uploaded", chunk.GetID(), chunkSize, chunkIndex, len(chunks))
|
||||
}
|
||||
|
||||
@@ -35,6 +35,19 @@ var DEFAULT_KEY = []byte("duplicacy")
|
||||
// standard zlib levels of -1 to 9.
|
||||
var DEFAULT_COMPRESSION_LEVEL = 100
|
||||
|
||||
// zstd compression levels starting from 200
|
||||
var ZSTD_COMPRESSION_LEVEL_FASTEST = 200
|
||||
var ZSTD_COMPRESSION_LEVEL_DEFAULT = 201
|
||||
var ZSTD_COMPRESSION_LEVEL_BETTER = 202
|
||||
var ZSTD_COMPRESSION_LEVEL_BEST = 203
|
||||
|
||||
var ZSTD_COMPRESSION_LEVELS = map[string]int {
|
||||
"fastest": ZSTD_COMPRESSION_LEVEL_FASTEST,
|
||||
"default": ZSTD_COMPRESSION_LEVEL_DEFAULT,
|
||||
"better": ZSTD_COMPRESSION_LEVEL_BETTER,
|
||||
"best": ZSTD_COMPRESSION_LEVEL_BEST,
|
||||
}
|
||||
|
||||
// The new banner of the config file (to differentiate from the old format where the salt and iterations are fixed)
|
||||
var CONFIG_BANNER = "duplicacy\001"
|
||||
|
||||
@@ -72,8 +85,8 @@ type Config struct {
|
||||
FileKey []byte `json:"-"`
|
||||
|
||||
// for erasure coding
|
||||
DataShards int `json:'data-shards'`
|
||||
ParityShards int `json:'parity-shards'`
|
||||
DataShards int `json:"data-shards"`
|
||||
ParityShards int `json:"parity-shards"`
|
||||
|
||||
// for RSA encryption
|
||||
rsaPrivateKey *rsa.PrivateKey
|
||||
@@ -156,10 +169,9 @@ func (config *Config) UnmarshalJSON(description []byte) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (config *Config) IsCompatiableWith(otherConfig *Config) bool {
|
||||
func (config *Config) IsCompatibleWith(otherConfig *Config) bool {
|
||||
|
||||
return config.CompressionLevel == otherConfig.CompressionLevel &&
|
||||
config.AverageChunkSize == otherConfig.AverageChunkSize &&
|
||||
return config.AverageChunkSize == otherConfig.AverageChunkSize &&
|
||||
config.MaximumChunkSize == otherConfig.MaximumChunkSize &&
|
||||
config.MinimumChunkSize == otherConfig.MinimumChunkSize &&
|
||||
bytes.Equal(config.ChunkSeed, otherConfig.ChunkSeed) &&
|
||||
@@ -202,6 +214,14 @@ func (config *Config) Print() {
|
||||
|
||||
}
|
||||
|
||||
func (config *Config) PrintCompressionLevel() {
|
||||
for name, level := range ZSTD_COMPRESSION_LEVELS {
|
||||
if level == config.CompressionLevel {
|
||||
LOG_INFO("COMPRESSION_LEVEL", "Zstd compression is enabled (level: %s)", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maximumChunkSize int, mininumChunkSize int,
|
||||
isEncrypted bool, copyFrom *Config, bitCopy bool) (config *Config) {
|
||||
|
||||
@@ -234,7 +254,6 @@ func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maxi
|
||||
}
|
||||
|
||||
if copyFrom != nil {
|
||||
config.CompressionLevel = copyFrom.CompressionLevel
|
||||
|
||||
config.AverageChunkSize = copyFrom.AverageChunkSize
|
||||
config.MaximumChunkSize = copyFrom.MaximumChunkSize
|
||||
@@ -244,6 +263,8 @@ func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maxi
|
||||
config.HashKey = copyFrom.HashKey
|
||||
|
||||
if bitCopy {
|
||||
config.CompressionLevel = copyFrom.CompressionLevel
|
||||
|
||||
config.IDKey = copyFrom.IDKey
|
||||
config.ChunkKey = copyFrom.ChunkKey
|
||||
config.FileKey = copyFrom.FileKey
|
||||
@@ -294,7 +315,10 @@ func (config *Config) PutChunk(chunk *Chunk) {
|
||||
}
|
||||
|
||||
func (config *Config) NewKeyedHasher(key []byte) hash.Hash {
|
||||
if config.CompressionLevel == DEFAULT_COMPRESSION_LEVEL {
|
||||
// Early versions of Duplicacy used SHA256 as the hash function for chunk IDs at the time when
|
||||
// only zlib compression was supported. Later SHA256 was replaced by Blake2b and LZ4 was used
|
||||
// for compression (with compression level set to 100).
|
||||
if config.CompressionLevel >= DEFAULT_COMPRESSION_LEVEL {
|
||||
hasher, err := blake2.New(&blake2.Config{Size: 32, Key: key})
|
||||
if err != nil {
|
||||
LOG_ERROR("HASH_KEY", "Invalid hash key: %x", key)
|
||||
@@ -339,7 +363,7 @@ func (hasher *DummyHasher) BlockSize() int {
|
||||
func (config *Config) NewFileHasher() hash.Hash {
|
||||
if SkipFileHash {
|
||||
return &DummyHasher{}
|
||||
} else if config.CompressionLevel == DEFAULT_COMPRESSION_LEVEL {
|
||||
} else if config.CompressionLevel >= DEFAULT_COMPRESSION_LEVEL {
|
||||
hasher, _ := blake2.New(&blake2.Config{Size: 32})
|
||||
return hasher
|
||||
} else {
|
||||
@@ -436,7 +460,7 @@ func DownloadConfig(storage Storage, password string) (config *Config, isEncrypt
|
||||
}
|
||||
|
||||
// Decrypt the config file. masterKey == nil means no encryption.
|
||||
err = configFile.Decrypt(masterKey, "")
|
||||
err, _ = configFile.Decrypt(masterKey, "")
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("Failed to retrieve the config file: %v", err)
|
||||
}
|
||||
|
||||
@@ -4,8 +4,11 @@
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
@@ -16,11 +19,13 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
|
||||
"github.com/vmihailenco/msgpack"
|
||||
"github.com/vmihailenco/msgpack"
|
||||
)
|
||||
|
||||
const (
|
||||
entryHardLinkRootChunkMarker = -9
|
||||
entryHardLinkTargetChunkMarker = -10
|
||||
)
|
||||
|
||||
// This is the hidden directory in the repository for storing various files.
|
||||
@@ -93,7 +98,7 @@ func CreateEntryFromFileInfo(fileInfo os.FileInfo, directory string) *Entry {
|
||||
Mode: uint32(mode),
|
||||
}
|
||||
|
||||
GetOwner(entry, &fileInfo)
|
||||
GetOwner(entry, fileInfo)
|
||||
|
||||
return entry
|
||||
}
|
||||
@@ -110,15 +115,45 @@ func (entry *Entry) Copy() *Entry {
|
||||
UID: entry.UID,
|
||||
GID: entry.GID,
|
||||
|
||||
StartChunk: entry.StartChunk,
|
||||
StartChunk: entry.StartChunk,
|
||||
StartOffset: entry.StartOffset,
|
||||
EndChunk: entry.EndChunk,
|
||||
EndOffset: entry.EndOffset,
|
||||
EndChunk: entry.EndChunk,
|
||||
EndOffset: entry.EndOffset,
|
||||
|
||||
Attributes: entry.Attributes,
|
||||
}
|
||||
}
|
||||
|
||||
func (entry *Entry) HardLinkTo(target *Entry, startChunk int, endChunk int) *Entry {
|
||||
endOffset := target.EndOffset
|
||||
link := entry.Link
|
||||
|
||||
if !target.IsFile() {
|
||||
startChunk = target.StartChunk
|
||||
endChunk = entry.EndChunk
|
||||
endOffset = entry.EndOffset
|
||||
link = target.Link
|
||||
}
|
||||
return &Entry{
|
||||
Path: entry.Path,
|
||||
Size: target.Size,
|
||||
Time: target.Time,
|
||||
Mode: target.Mode,
|
||||
Link: link,
|
||||
Hash: target.Hash,
|
||||
|
||||
UID: target.UID,
|
||||
GID: target.GID,
|
||||
|
||||
StartChunk: startChunk,
|
||||
StartOffset: target.StartOffset,
|
||||
EndChunk: endChunk,
|
||||
EndOffset: endOffset,
|
||||
|
||||
Attributes: target.Attributes,
|
||||
}
|
||||
}
|
||||
|
||||
// CreateEntryFromJSON creates an entry from a json description.
|
||||
func (entry *Entry) UnmarshalJSON(description []byte) (err error) {
|
||||
|
||||
@@ -362,12 +397,12 @@ func (entry *Entry) EncodeMsgpack(encoder *msgpack.Encoder) error {
|
||||
|
||||
if entry.Attributes != nil {
|
||||
attributes := make([]string, numberOfAttributes)
|
||||
i := 0
|
||||
for attribute := range *entry.Attributes {
|
||||
attributes[i] = attribute
|
||||
i++
|
||||
}
|
||||
sort.Strings(attributes)
|
||||
i := 0
|
||||
for attribute := range *entry.Attributes {
|
||||
attributes[i] = attribute
|
||||
i++
|
||||
}
|
||||
sort.Strings(attributes)
|
||||
for _, attribute := range attributes {
|
||||
err = encoder.EncodeString(attribute)
|
||||
if err != nil {
|
||||
@@ -380,7 +415,7 @@ func (entry *Entry) EncodeMsgpack(encoder *msgpack.Encoder) error {
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) DecodeMsgpack(decoder *msgpack.Decoder) error {
|
||||
@@ -488,18 +523,42 @@ func (entry *Entry) IsLink() bool {
|
||||
return entry.Mode&uint32(os.ModeSymlink) != 0
|
||||
}
|
||||
|
||||
func (entry *Entry) IsSpecial() bool {
|
||||
return entry.Mode&uint32(os.ModeNamedPipe|os.ModeDevice|os.ModeCharDevice|os.ModeSocket) != 0
|
||||
}
|
||||
|
||||
func (entry *Entry) IsComplete() bool {
|
||||
return entry.Size >= 0
|
||||
}
|
||||
|
||||
func (entry *Entry) IsHardLinkChild() bool {
|
||||
return (entry.IsFile() && len(entry.Link) > 0 && entry.Link != "/") || (!entry.IsDir() && entry.EndChunk == entryHardLinkTargetChunkMarker)
|
||||
}
|
||||
|
||||
func (entry *Entry) IsHardLinkRoot() bool {
|
||||
return (entry.IsFile() && entry.Link == "/") || (!entry.IsDir() && entry.EndChunk == entryHardLinkRootChunkMarker)
|
||||
}
|
||||
|
||||
func (entry *Entry) GetHardLinkId() (int, error) {
|
||||
if entry.IsFile() {
|
||||
i, err := strconv.ParseUint(entry.Link, 16, 64)
|
||||
return int(i), err
|
||||
} else {
|
||||
if entry.EndChunk != entryHardLinkTargetChunkMarker {
|
||||
return 0, errors.New("Entry not marked as hard link child")
|
||||
}
|
||||
return entry.EndOffset, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (entry *Entry) GetPermissions() os.FileMode {
|
||||
return os.FileMode(entry.Mode) & fileModeMask
|
||||
}
|
||||
|
||||
func (entry *Entry) GetParent() string {
|
||||
path := entry.Path
|
||||
if path != "" && path[len(path) - 1] == '/' {
|
||||
path = path[:len(path) - 1]
|
||||
if path != "" && path[len(path)-1] == '/' {
|
||||
path = path[:len(path)-1]
|
||||
}
|
||||
i := strings.LastIndex(path, "/")
|
||||
if i == -1 {
|
||||
@@ -523,45 +582,64 @@ func (entry *Entry) String(maxSizeDigits int) string {
|
||||
return fmt.Sprintf("%*d %s %64s %s", maxSizeDigits, entry.Size, modifiedTime, entry.Hash, entry.Path)
|
||||
}
|
||||
|
||||
func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setOwner bool) bool {
|
||||
type RestoreMetadataOptions struct {
|
||||
SetOwner bool
|
||||
ExcludeXattrs bool
|
||||
NormalizeXattrs bool
|
||||
IncludeFileFlags bool
|
||||
FileFlagsMask uint32
|
||||
}
|
||||
|
||||
func (entry *Entry) RestoreMetadata(fullPath string, fileInfo os.FileInfo,
|
||||
options RestoreMetadataOptions) bool {
|
||||
|
||||
if fileInfo == nil {
|
||||
stat, err := os.Lstat(fullPath)
|
||||
fileInfo = &stat
|
||||
var err error
|
||||
fileInfo, err = os.Lstat(fullPath)
|
||||
if err != nil {
|
||||
LOG_ERROR("RESTORE_STAT", "Failed to retrieve the file info: %v", err)
|
||||
LOG_ERROR("RESTORE_STAT", "Failed to retrieve the file info on %s: %v", entry.Path, err)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if !options.ExcludeXattrs {
|
||||
err := entry.SetAttributesToFile(fullPath, options.NormalizeXattrs)
|
||||
if err != nil {
|
||||
LOG_WARN("RESTORE_ATTR", "Failed to set extended attributes on %s: %v", entry.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Note that chown can remove setuid/setgid bits so should be called before chmod
|
||||
if setOwner {
|
||||
if options.SetOwner {
|
||||
if !SetOwner(fullPath, entry, fileInfo) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Only set the permission if the file is not a symlink
|
||||
if !entry.IsLink() && (*fileInfo).Mode()&fileModeMask != entry.GetPermissions() {
|
||||
if !entry.IsLink() && fileInfo.Mode()&fileModeMask != entry.GetPermissions() {
|
||||
err := os.Chmod(fullPath, entry.GetPermissions())
|
||||
if err != nil {
|
||||
LOG_ERROR("RESTORE_CHMOD", "Failed to set the file permissions: %v", err)
|
||||
LOG_ERROR("RESTORE_CHMOD", "Failed to set the file permissions on %s: %v", entry.Path, err)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Only set the time if the file is not a symlink
|
||||
if !entry.IsLink() && (*fileInfo).ModTime().Unix() != entry.Time {
|
||||
if !entry.IsLink() && fileInfo.ModTime().Unix() != entry.Time {
|
||||
modifiedTime := time.Unix(entry.Time, 0)
|
||||
err := os.Chtimes(fullPath, modifiedTime, modifiedTime)
|
||||
if err != nil {
|
||||
LOG_ERROR("RESTORE_CHTIME", "Failed to set the modification time: %v", err)
|
||||
LOG_ERROR("RESTORE_CHTIME", "Failed to set the modification time on %s: %v", entry.Path, err)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if entry.Attributes != nil && len(*entry.Attributes) > 0 {
|
||||
entry.SetAttributesToFile(fullPath)
|
||||
if options.IncludeFileFlags {
|
||||
err := entry.RestoreLateFileFlags(fullPath, fileInfo, options.FileFlagsMask)
|
||||
if err != nil {
|
||||
LOG_WARN("RESTORE_FLAGS", "Failed to set file flags on %s: %v", entry.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
@@ -596,7 +674,7 @@ func ComparePaths(left string, right string) int {
|
||||
for i := p; i < len(left); i++ {
|
||||
c3 = left[i]
|
||||
if c3 == '/' {
|
||||
last1 = i == len(left) - 1
|
||||
last1 = i == len(left)-1
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -606,7 +684,7 @@ func ComparePaths(left string, right string) int {
|
||||
for i := p; i < len(right); i++ {
|
||||
c4 = right[i]
|
||||
if c4 == '/' {
|
||||
last2 = i == len(right) - 1
|
||||
last2 = i == len(right)-1
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -694,10 +772,39 @@ func (files FileInfoCompare) Less(i, j int) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// ListEntries returns a list of entries representing file and subdirectories under the directory 'path'. Entry paths
|
||||
// are normalized as relative to 'top'. 'patterns' are used to exclude or include certain files.
|
||||
func ListEntries(top string, path string, patterns []string, nobackupFile string, excludeByAttribute bool, listingChannel chan *Entry) (directoryList []*Entry,
|
||||
skippedFiles []string, err error) {
|
||||
type EntryListerOptions struct {
|
||||
Patterns []string
|
||||
NoBackupFile string
|
||||
ExcludeByAttribute bool
|
||||
ExcludeXattrs bool
|
||||
NormalizeXattr bool
|
||||
IncludeFileFlags bool
|
||||
IncludeSpecials bool
|
||||
}
|
||||
|
||||
type EntryLister interface {
|
||||
ListDir(top string, path string, listingChannel chan *Entry, options *EntryListerOptions) (directoryList []*Entry, skippedFiles []string, err error)
|
||||
}
|
||||
|
||||
type LocalDirectoryLister struct {
|
||||
linkIndex int
|
||||
linkTable map[listEntryLinkKey]int // map unique inode details to initially found path
|
||||
}
|
||||
|
||||
func NewLocalDirectoryLister() *LocalDirectoryLister {
|
||||
return &LocalDirectoryLister{
|
||||
linkTable: make(map[listEntryLinkKey]int),
|
||||
}
|
||||
}
|
||||
|
||||
// ListDir returns a list of entries representing file and subdirectories under the directory 'path'.
|
||||
// Entry paths are normalized as relative to 'top'.
|
||||
func (dl *LocalDirectoryLister) ListDir(top string, path string, listingChannel chan *Entry,
|
||||
options *EntryListerOptions) (directoryList []*Entry, skippedFiles []string, err error) {
|
||||
|
||||
if options == nil {
|
||||
options = &EntryListerOptions{}
|
||||
}
|
||||
|
||||
LOG_DEBUG("LIST_ENTRIES", "Listing %s", path)
|
||||
|
||||
@@ -710,10 +817,12 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
|
||||
return directoryList, nil, err
|
||||
}
|
||||
|
||||
patterns := options.Patterns
|
||||
|
||||
// This binary search works because ioutil.ReadDir returns files sorted by Name() by default
|
||||
if nobackupFile != "" {
|
||||
ii := sort.Search(len(files), func(ii int) bool { return strings.Compare(files[ii].Name(), nobackupFile) >= 0 })
|
||||
if ii < len(files) && files[ii].Name() == nobackupFile {
|
||||
if options.NoBackupFile != "" {
|
||||
ii := sort.Search(len(files), func(ii int) bool { return strings.Compare(files[ii].Name(), options.NoBackupFile) >= 0 })
|
||||
if ii < len(files) && files[ii].Name() == options.NoBackupFile {
|
||||
LOG_DEBUG("LIST_NOBACKUP", "%s is excluded due to nobackup file", path)
|
||||
return directoryList, skippedFiles, nil
|
||||
}
|
||||
@@ -735,13 +844,44 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
|
||||
if f.Name() == DUPLICACY_DIRECTORY {
|
||||
continue
|
||||
}
|
||||
|
||||
entry := CreateEntryFromFileInfo(f, normalizedPath)
|
||||
if len(patterns) > 0 && !MatchPath(entry.Path, patterns) {
|
||||
continue
|
||||
}
|
||||
|
||||
linkKey, isHardLinked := entry.getHardLinkKey(f)
|
||||
if isHardLinked {
|
||||
if linkIndex, seen := dl.linkTable[linkKey]; seen {
|
||||
if linkIndex == -1 {
|
||||
LOG_DEBUG("LIST_EXCLUDE", "%s was excluded or skipped (hard link)", entry.Path)
|
||||
continue
|
||||
}
|
||||
|
||||
entry.Size = 0
|
||||
if entry.IsFile() {
|
||||
entry.Link = strconv.FormatInt(int64(linkIndex), 16)
|
||||
} else {
|
||||
entry.EndChunk = entryHardLinkTargetChunkMarker
|
||||
entry.EndOffset = linkIndex
|
||||
}
|
||||
listingChannel <- entry
|
||||
continue
|
||||
} else {
|
||||
if entry.IsFile() {
|
||||
entry.Link = "/"
|
||||
} else {
|
||||
entry.EndChunk = entryHardLinkRootChunkMarker
|
||||
}
|
||||
dl.linkTable[linkKey] = -1
|
||||
}
|
||||
}
|
||||
|
||||
fullPath := joinPath(top, entry.Path)
|
||||
|
||||
if entry.IsLink() {
|
||||
isRegular := false
|
||||
isRegular, entry.Link, err = Readlink(joinPath(top, entry.Path))
|
||||
isRegular, entry.Link, err = Readlink(fullPath)
|
||||
if err != nil {
|
||||
LOG_WARN("LIST_LINK", "Failed to read the symlink %s: %v", entry.Path, err)
|
||||
skippedFiles = append(skippedFiles, entry.Path)
|
||||
@@ -751,7 +891,7 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
|
||||
if isRegular {
|
||||
entry.Mode ^= uint32(os.ModeSymlink)
|
||||
} else if path == "" && (filepath.IsAbs(entry.Link) || filepath.HasPrefix(entry.Link, `\\`)) && !strings.HasPrefix(entry.Link, normalizedTop) {
|
||||
stat, err := os.Stat(joinPath(top, entry.Path))
|
||||
stat, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
LOG_WARN("LIST_LINK", "Failed to read the symlink: %v", err)
|
||||
skippedFiles = append(skippedFiles, entry.Path)
|
||||
@@ -769,19 +909,35 @@ func ListEntries(top string, path string, patterns []string, nobackupFile string
|
||||
}
|
||||
entry = newEntry
|
||||
}
|
||||
} else if options.IncludeSpecials && entry.IsSpecial() {
|
||||
if err := entry.ReadSpecial(fullPath, f); err != nil {
|
||||
LOG_WARN("LIST_DEV", "Failed to save device node %s: %v", entry.Path, err)
|
||||
skippedFiles = append(skippedFiles, entry.Path)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
entry.ReadAttributes(top)
|
||||
if !options.ExcludeXattrs {
|
||||
if err := entry.ReadAttributes(f, fullPath, false); err != nil {
|
||||
LOG_WARN("LIST_ATTR", "Failed to read xattrs on %s: %v", entry.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
if excludeByAttribute && entry.Attributes != nil && excludedByAttribute(*entry.Attributes) {
|
||||
// if the flags are already in the FileInfo we can keep them
|
||||
if !entry.GetFileFlags(f) && options.IncludeFileFlags {
|
||||
if err := entry.ReadFileFlags(f, fullPath); err != nil {
|
||||
LOG_WARN("LIST_ATTR", "Failed to read file flags on %s: %v", entry.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
if options.ExcludeByAttribute && entry.Attributes != nil && excludedByAttribute(*entry.Attributes) {
|
||||
LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute", entry.Path)
|
||||
continue
|
||||
}
|
||||
|
||||
if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
|
||||
LOG_WARN("LIST_SKIP", "Skipped non-regular file %s", entry.Path)
|
||||
skippedFiles = append(skippedFiles, entry.Path)
|
||||
continue
|
||||
if isHardLinked {
|
||||
dl.linkTable[linkKey] = dl.linkIndex
|
||||
dl.linkIndex++
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -13,11 +14,10 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/gilbertchen/xattr"
|
||||
"github.com/vmihailenco/msgpack"
|
||||
"github.com/pkg/xattr"
|
||||
|
||||
"github.com/vmihailenco/msgpack"
|
||||
)
|
||||
|
||||
func TestEntrySort(t *testing.T) {
|
||||
@@ -165,23 +165,25 @@ func TestEntryOrder(t *testing.T) {
|
||||
continue
|
||||
}
|
||||
|
||||
err := ioutil.WriteFile(fullPath, []byte(file), 0700)
|
||||
err := os.WriteFile(fullPath, []byte(file), 0700)
|
||||
if err != nil {
|
||||
t.Errorf("WriteFile(%s) returned an error: %s", fullPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
lister := NewLocalDirectoryLister()
|
||||
|
||||
directories := make([]*Entry, 0, 4)
|
||||
directories = append(directories, CreateEntry("", 0, 0, 0))
|
||||
|
||||
entries := make([]*Entry, 0, 4)
|
||||
entryChannel := make(chan *Entry, 1024)
|
||||
entryChannel := make(chan *Entry, 1024)
|
||||
entries = append(entries, CreateEntry("", 0, 0, 0))
|
||||
|
||||
for len(directories) > 0 {
|
||||
directory := directories[len(directories)-1]
|
||||
directories = directories[:len(directories)-1]
|
||||
subdirectories, _, err := ListEntries(testDir, directory.Path, nil, "", false, entryChannel)
|
||||
subdirectories, _, err := lister.ListDir(testDir, directory.Path, entryChannel, nil)
|
||||
if err != nil {
|
||||
t.Errorf("ListEntries(%s, %s) returned an error: %s", testDir, directory.Path, err)
|
||||
}
|
||||
@@ -233,8 +235,18 @@ func TestEntryOrder(t *testing.T) {
|
||||
// TestEntryExcludeByAttribute tests the excludeByAttribute parameter to the ListEntries function
|
||||
func TestEntryExcludeByAttribute(t *testing.T) {
|
||||
|
||||
if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") {
|
||||
t.Skip("skipping test not darwin or linux")
|
||||
var excludeAttrName string
|
||||
var excludeAttrValue []byte
|
||||
|
||||
if runtime.GOOS == "darwin" {
|
||||
excludeAttrName = "com.apple.metadata:com_apple_backup_excludeItem"
|
||||
excludeAttrValue = []byte("com.apple.backupd")
|
||||
} else if runtime.GOOS == "linux" {
|
||||
excludeAttrName = "user.duplicacy_exclude"
|
||||
} else if runtime.GOOS == "freebsd" || runtime.GOOS == "netbsd" {
|
||||
excludeAttrName = "duplicacy_exclude"
|
||||
} else {
|
||||
t.Skip("skipping test, not darwin, linux, freebsd, or netbsd")
|
||||
}
|
||||
|
||||
testDir := filepath.Join(os.TempDir(), "duplicacy_test")
|
||||
@@ -264,7 +276,7 @@ func TestEntryExcludeByAttribute(t *testing.T) {
|
||||
continue
|
||||
}
|
||||
|
||||
err := ioutil.WriteFile(fullPath, []byte(file), 0700)
|
||||
err := os.WriteFile(fullPath, []byte(file), 0700)
|
||||
if err != nil {
|
||||
t.Errorf("WriteFile(%s) returned an error: %s", fullPath, err)
|
||||
}
|
||||
@@ -273,12 +285,14 @@ func TestEntryExcludeByAttribute(t *testing.T) {
|
||||
for _, file := range DATA {
|
||||
fullPath := filepath.Join(testDir, file)
|
||||
if strings.Contains(file, "exclude") {
|
||||
xattr.Setxattr(fullPath, "com.apple.metadata:com_apple_backup_excludeItem", []byte("com.apple.backupd"))
|
||||
xattr.Set(fullPath, excludeAttrName, excludeAttrValue)
|
||||
}
|
||||
}
|
||||
|
||||
for _, excludeByAttribute := range [2]bool{true, false} {
|
||||
t.Logf("testing excludeByAttribute: %t", excludeByAttribute)
|
||||
|
||||
lister := NewLocalDirectoryLister()
|
||||
directories := make([]*Entry, 0, 4)
|
||||
directories = append(directories, CreateEntry("", 0, 0, 0))
|
||||
|
||||
@@ -289,7 +303,11 @@ func TestEntryExcludeByAttribute(t *testing.T) {
|
||||
for len(directories) > 0 {
|
||||
directory := directories[len(directories)-1]
|
||||
directories = directories[:len(directories)-1]
|
||||
subdirectories, _, err := ListEntries(testDir, directory.Path, nil, "", excludeByAttribute, entryChannel)
|
||||
subdirectories, _, err := lister.ListDir(testDir, directory.Path, entryChannel,
|
||||
&EntryListerOptions{
|
||||
ExcludeByAttribute: excludeByAttribute,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("ListEntries(%s, %s) returned an error: %s", testDir, directory.Path, err)
|
||||
}
|
||||
@@ -372,4 +390,4 @@ func TestEntryEncoding(t *testing.T) {
|
||||
t.Error("Decoded entry is different than the original one")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,12 +111,12 @@ func (entryList *EntryList)createOnDiskFile() error {
|
||||
// Add an entry to the entry list
|
||||
func (entryList *EntryList)AddEntry(entry *Entry) error {
|
||||
|
||||
if !entry.IsDir() && !entry.IsLink() {
|
||||
if entry.IsFile() {
|
||||
entryList.NumberOfEntries++
|
||||
}
|
||||
|
||||
if !entry.IsComplete() {
|
||||
if entry.IsDir() || entry.IsLink() {
|
||||
if !entry.IsFile() {
|
||||
entry.Size = 0
|
||||
} else {
|
||||
modifiedEntry := ModifiedEntry {
|
||||
@@ -550,7 +550,7 @@ func loadIncompleteSnapshot(snapshotID string, cachePath string) *EntryList {
|
||||
}
|
||||
}
|
||||
|
||||
LOG_INFO("INCOMPLETE_LOAD", "Previous incomlete backup contains %d files and %d chunks",
|
||||
LOG_INFO("INCOMPLETE_LOAD", "Previous incomplete backup contains %d files and %d chunks",
|
||||
entryList.NumberOfEntries, len(entryList.PreservedChunkLengths) + len(entryList.UploadedChunkHashes))
|
||||
|
||||
return entryList
|
||||
@@ -571,4 +571,4 @@ func deleteIncompleteSnapshot(cachePath string) {
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -775,7 +775,7 @@ func (storage *GCDStorage) GetFileInfo(threadIndex int, filePath string) (exist
|
||||
// DownloadFile reads the file at 'filePath' into the chunk.
|
||||
func (storage *GCDStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
|
||||
// We never download the fossil so there is no need to convert the path
|
||||
fileID, err := storage.getIDFromPath(threadIndex, filePath, false)
|
||||
fileID, err := storage.getIDFromPath(threadIndex, storage.convertFilePath(filePath), false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
crypto_rand "crypto/rand"
|
||||
"math/rand"
|
||||
)
|
||||
|
||||
func TestHubicClient(t *testing.T) {
|
||||
|
||||
hubicClient, err := NewHubicClient("hubic-token.json")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create the Hubic client: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
hubicClient.TestMode = true
|
||||
|
||||
existingFiles, err := hubicClient.ListEntries("")
|
||||
for _, file := range existingFiles {
|
||||
fmt.Printf("name: %s, isDir: %t\n", file.Name, file.Type == "application/directory")
|
||||
}
|
||||
|
||||
testExists, _, _, err := hubicClient.GetFileInfo("test")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list the test directory: %v", err)
|
||||
return
|
||||
}
|
||||
if !testExists {
|
||||
err = hubicClient.CreateDirectory("test")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create the test directory: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
test1Exists, _, _, err := hubicClient.GetFileInfo("test/test1")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list the test1 directory: %v", err)
|
||||
return
|
||||
}
|
||||
if !test1Exists {
|
||||
err = hubicClient.CreateDirectory("test/test1")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create the test1 directory: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
test2Exists, _, _, err := hubicClient.GetFileInfo("test/test2")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list the test2 directory: %v", err)
|
||||
return
|
||||
}
|
||||
if !test2Exists {
|
||||
err = hubicClient.CreateDirectory("test/test2")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create the test2 directory: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
numberOfFiles := 20
|
||||
maxFileSize := 64 * 1024
|
||||
|
||||
for i := 0; i < numberOfFiles; i++ {
|
||||
content := make([]byte, rand.Int()%maxFileSize+1)
|
||||
_, err = crypto_rand.Read(content)
|
||||
if err != nil {
|
||||
t.Errorf("Error generating random content: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
hasher := sha256.New()
|
||||
hasher.Write(content)
|
||||
filename := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
fmt.Printf("file: %s\n", filename)
|
||||
|
||||
err = hubicClient.UploadFile("test/test1/"+filename, content, 100)
|
||||
if err != nil {
|
||||
/*if e, ok := err.(ACDError); !ok || e.Status != 409 */ {
|
||||
t.Errorf("Failed to upload the file %s: %v", filename, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entries, err := hubicClient.ListEntries("test/test1")
|
||||
if err != nil {
|
||||
t.Errorf("Error list randomly generated files: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
|
||||
exists, isDir, size, err := hubicClient.GetFileInfo("test/test1/" + entry.Name)
|
||||
fmt.Printf("%s exists: %t, isDir: %t, size: %d, err: %v\n", "test/test1/"+entry.Name, exists, isDir, size, err)
|
||||
|
||||
err = hubicClient.MoveFile("test/test1/"+entry.Name, "test/test2/"+entry.Name)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to move %s: %v", entry.Name, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
entries, err = hubicClient.ListEntries("test/test2")
|
||||
if err != nil {
|
||||
t.Errorf("Error list randomly generated files: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
readCloser, _, err := hubicClient.DownloadFile("test/test2/" + entry.Name)
|
||||
if err != nil {
|
||||
t.Errorf("Error downloading file %s: %v", entry.Name, err)
|
||||
return
|
||||
}
|
||||
|
||||
hasher := sha256.New()
|
||||
io.Copy(hasher, readCloser)
|
||||
hash := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
if hash != entry.Name {
|
||||
t.Errorf("File %s, hash %s", entry.Name, hash)
|
||||
}
|
||||
|
||||
readCloser.Close()
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
|
||||
err = hubicClient.DeleteFile("test/test2/" + entry.Name)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to delete the file %s: %v", entry.Name, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -6,7 +6,7 @@ package duplicacy
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
@@ -86,7 +86,7 @@ func keyringGet(key string) (value string) {
|
||||
return ""
|
||||
}
|
||||
|
||||
description, err := ioutil.ReadFile(keyringFile)
|
||||
description, err := os.ReadFile(keyringFile)
|
||||
if err != nil {
|
||||
LOG_DEBUG("KEYRING_READ", "Keyring file not read: %v", err)
|
||||
return ""
|
||||
@@ -125,7 +125,7 @@ func keyringSet(key string, value string) bool {
|
||||
|
||||
keyring := make(map[string][]byte)
|
||||
|
||||
description, err := ioutil.ReadFile(keyringFile)
|
||||
description, err := os.ReadFile(keyringFile)
|
||||
if err == nil {
|
||||
err = json.Unmarshal(description, &keyring)
|
||||
if err != nil {
|
||||
@@ -160,7 +160,7 @@ func keyringSet(key string, value string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(keyringFile, description, 0600)
|
||||
err = os.WriteFile(keyringFile, description, 0600)
|
||||
if err != nil {
|
||||
LOG_DEBUG("KEYRING_WRITE", "Failed to save the keyring storage to file %s: %v", keyringFile, err)
|
||||
return false
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -39,6 +40,7 @@ type OneDriveClient struct {
|
||||
|
||||
TokenFile string
|
||||
Token *oauth2.Token
|
||||
OAConfig *oauth2.Config
|
||||
TokenLock *sync.Mutex
|
||||
|
||||
IsConnected bool
|
||||
@@ -49,7 +51,7 @@ type OneDriveClient struct {
|
||||
APIURL string
|
||||
}
|
||||
|
||||
func NewOneDriveClient(tokenFile string, isBusiness bool) (*OneDriveClient, error) {
|
||||
func NewOneDriveClient(tokenFile string, isBusiness bool, client_id string, client_secret string, drive_id string) (*OneDriveClient, error) {
|
||||
|
||||
description, err := ioutil.ReadFile(tokenFile)
|
||||
if err != nil {
|
||||
@@ -65,16 +67,34 @@ func NewOneDriveClient(tokenFile string, isBusiness bool) (*OneDriveClient, erro
|
||||
HTTPClient: http.DefaultClient,
|
||||
TokenFile: tokenFile,
|
||||
Token: token,
|
||||
OAConfig: nil,
|
||||
TokenLock: &sync.Mutex{},
|
||||
IsBusiness: isBusiness,
|
||||
}
|
||||
|
||||
if (client_id != "") {
|
||||
oneOauthConfig := oauth2.Config{
|
||||
ClientID: client_id,
|
||||
ClientSecret: client_secret,
|
||||
Scopes: []string{"Files.ReadWrite", "offline_access"},
|
||||
Endpoint: oauth2.Endpoint{
|
||||
AuthURL: "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
|
||||
TokenURL: "https://login.microsoftonline.com/common/oauth2/v2.0/token",
|
||||
},
|
||||
}
|
||||
|
||||
client.OAConfig = &oneOauthConfig
|
||||
}
|
||||
|
||||
if isBusiness {
|
||||
client.RefreshTokenURL = "https://duplicacy.com/odb_refresh"
|
||||
client.APIURL = "https://graph.microsoft.com/v1.0/me"
|
||||
client.APIURL = "https://graph.microsoft.com/v1.0/me/drive"
|
||||
if drive_id != "" {
|
||||
client.APIURL = "https://graph.microsoft.com/v1.0/drives/"+drive_id
|
||||
}
|
||||
} else {
|
||||
client.RefreshTokenURL = "https://duplicacy.com/one_refresh"
|
||||
client.APIURL = "https://api.onedrive.com/v1.0"
|
||||
client.APIURL = "https://api.onedrive.com/v1.0/drive"
|
||||
}
|
||||
|
||||
client.RefreshToken(false)
|
||||
@@ -218,15 +238,25 @@ func (client *OneDriveClient) RefreshToken(force bool) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
readCloser, _, err := client.call(client.RefreshTokenURL, "POST", client.Token, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to refresh the access token: %v", err)
|
||||
}
|
||||
if (client.OAConfig == nil) {
|
||||
readCloser, _, err := client.call(client.RefreshTokenURL, "POST", client.Token, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to refresh the access token: %v", err)
|
||||
}
|
||||
|
||||
defer readCloser.Close()
|
||||
defer readCloser.Close()
|
||||
|
||||
if err = json.NewDecoder(readCloser).Decode(client.Token); err != nil {
|
||||
return err
|
||||
if err = json.NewDecoder(readCloser).Decode(client.Token); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
ctx := context.Background()
|
||||
tokenSource := client.OAConfig.TokenSource(ctx, client.Token)
|
||||
token, err := tokenSource.Token()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to refresh the access token: %v", err)
|
||||
}
|
||||
client.Token = token
|
||||
}
|
||||
|
||||
description, err := json.Marshal(client.Token)
|
||||
@@ -258,9 +288,9 @@ func (client *OneDriveClient) ListEntries(path string) ([]OneDriveEntry, error)
|
||||
|
||||
entries := []OneDriveEntry{}
|
||||
|
||||
url := client.APIURL + "/drive/root:/" + path + ":/children"
|
||||
url := client.APIURL + "/root:/" + path + ":/children"
|
||||
if path == "" {
|
||||
url = client.APIURL + "/drive/root/children"
|
||||
url = client.APIURL + "/root/children"
|
||||
}
|
||||
if client.TestMode {
|
||||
url += "?top=8"
|
||||
@@ -296,7 +326,8 @@ func (client *OneDriveClient) ListEntries(path string) ([]OneDriveEntry, error)
|
||||
|
||||
func (client *OneDriveClient) GetFileInfo(path string) (string, bool, int64, error) {
|
||||
|
||||
url := client.APIURL + "/drive/root:/" + path
|
||||
url := client.APIURL + "/root:/" + path
|
||||
if path == "" { url = client.APIURL + "/root" }
|
||||
url += "?select=id,name,size,folder"
|
||||
|
||||
readCloser, _, err := client.call(url, "GET", 0, "")
|
||||
@@ -321,7 +352,7 @@ func (client *OneDriveClient) GetFileInfo(path string) (string, bool, int64, err
|
||||
|
||||
func (client *OneDriveClient) DownloadFile(path string) (io.ReadCloser, int64, error) {
|
||||
|
||||
url := client.APIURL + "/drive/items/root:/" + path + ":/content"
|
||||
url := client.APIURL + "/items/root:/" + path + ":/content"
|
||||
|
||||
return client.call(url, "GET", 0, "")
|
||||
}
|
||||
@@ -331,7 +362,7 @@ func (client *OneDriveClient) UploadFile(path string, content []byte, rateLimit
|
||||
// Upload file using the simple method; this is only possible for OneDrive Personal or if the file
|
||||
// is smaller than 4MB for OneDrive Business
|
||||
if !client.IsBusiness || (client.TestMode && rand.Int() % 2 == 0) {
|
||||
url := client.APIURL + "/drive/root:/" + path + ":/content"
|
||||
url := client.APIURL + "/root:/" + path + ":/content"
|
||||
|
||||
readCloser, _, err := client.call(url, "PUT", CreateRateLimitedReader(content, rateLimit), "application/octet-stream")
|
||||
if err != nil {
|
||||
@@ -365,7 +396,7 @@ func (client *OneDriveClient) CreateUploadSession(path string) (uploadURL string
|
||||
},
|
||||
}
|
||||
|
||||
readCloser, _, err := client.call(client.APIURL + "/drive/root:/" + path + ":/createUploadSession", "POST", input, "application/json")
|
||||
readCloser, _, err := client.call(client.APIURL + "/root:/" + path + ":/createUploadSession", "POST", input, "application/json")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -409,7 +440,7 @@ func (client *OneDriveClient) UploadFileSession(uploadURL string, content []byte
|
||||
|
||||
func (client *OneDriveClient) DeleteFile(path string) error {
|
||||
|
||||
url := client.APIURL + "/drive/root:/" + path
|
||||
url := client.APIURL + "/root:/" + path
|
||||
|
||||
readCloser, _, err := client.call(url, "DELETE", 0, "")
|
||||
if err != nil {
|
||||
@@ -422,10 +453,10 @@ func (client *OneDriveClient) DeleteFile(path string) error {
|
||||
|
||||
func (client *OneDriveClient) MoveFile(path string, parent string) error {
|
||||
|
||||
url := client.APIURL + "/drive/root:/" + path
|
||||
url := client.APIURL + "/root:/" + path
|
||||
|
||||
parentReference := make(map[string]string)
|
||||
parentReference["path"] = "/drive/root:/" + parent
|
||||
parentReference["path"] = "/root:/" + parent
|
||||
|
||||
parameters := make(map[string]interface{})
|
||||
parameters["parentReference"] = parentReference
|
||||
@@ -477,7 +508,7 @@ func (client *OneDriveClient) CreateDirectory(path string, name string) error {
|
||||
return fmt.Errorf("The path '%s' is not a directory", path)
|
||||
}
|
||||
|
||||
url = client.APIURL + "/drive/root:/" + path + ":/children"
|
||||
url = client.APIURL + "/root:/" + path + ":/children"
|
||||
}
|
||||
|
||||
parameters := make(map[string]interface{})
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
crypto_rand "crypto/rand"
|
||||
"math/rand"
|
||||
)
|
||||
|
||||
func TestOneDriveClient(t *testing.T) {
|
||||
|
||||
oneDriveClient, err := NewOneDriveClient("one-token.json", false)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create the OneDrive client: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
oneDriveClient.TestMode = true
|
||||
|
||||
existingFiles, err := oneDriveClient.ListEntries("")
|
||||
for _, file := range existingFiles {
|
||||
fmt.Printf("name: %s, isDir: %t\n", file.Name, len(file.Folder) != 0)
|
||||
}
|
||||
|
||||
testID, _, _, err := oneDriveClient.GetFileInfo("test")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list the test directory: %v", err)
|
||||
return
|
||||
}
|
||||
if testID == "" {
|
||||
err = oneDriveClient.CreateDirectory("", "test")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create the test directory: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
test1ID, _, _, err := oneDriveClient.GetFileInfo("test/test1")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list the test1 directory: %v", err)
|
||||
return
|
||||
}
|
||||
if test1ID == "" {
|
||||
err = oneDriveClient.CreateDirectory("test", "test1")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create the test1 directory: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
test2ID, _, _, err := oneDriveClient.GetFileInfo("test/test2")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list the test2 directory: %v", err)
|
||||
return
|
||||
}
|
||||
if test2ID == "" {
|
||||
err = oneDriveClient.CreateDirectory("test", "test2")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create the test2 directory: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
numberOfFiles := 20
|
||||
maxFileSize := 64 * 1024
|
||||
|
||||
for i := 0; i < numberOfFiles; i++ {
|
||||
content := make([]byte, rand.Int()%maxFileSize+1)
|
||||
_, err = crypto_rand.Read(content)
|
||||
if err != nil {
|
||||
t.Errorf("Error generating random content: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
hasher := sha256.New()
|
||||
hasher.Write(content)
|
||||
filename := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
fmt.Printf("file: %s\n", filename)
|
||||
|
||||
err = oneDriveClient.UploadFile("test/test1/"+filename, content, 100)
|
||||
if err != nil {
|
||||
/*if e, ok := err.(ACDError); !ok || e.Status != 409 */ {
|
||||
t.Errorf("Failed to upload the file %s: %v", filename, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entries, err := oneDriveClient.ListEntries("test/test1")
|
||||
if err != nil {
|
||||
t.Errorf("Error list randomly generated files: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
err = oneDriveClient.MoveFile("test/test1/"+entry.Name, "test/test2")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to move %s: %v", entry.Name, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
entries, err = oneDriveClient.ListEntries("test/test2")
|
||||
if err != nil {
|
||||
t.Errorf("Error list randomly generated files: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
readCloser, _, err := oneDriveClient.DownloadFile("test/test2/" + entry.Name)
|
||||
if err != nil {
|
||||
t.Errorf("Error downloading file %s: %v", entry.Name, err)
|
||||
return
|
||||
}
|
||||
|
||||
hasher := sha256.New()
|
||||
io.Copy(hasher, readCloser)
|
||||
hash := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
if hash != entry.Name {
|
||||
t.Errorf("File %s, hash %s", entry.Name, hash)
|
||||
}
|
||||
|
||||
readCloser.Close()
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
|
||||
err = oneDriveClient.DeleteFile("test/test2/" + entry.Name)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to delete the file %s: %v", entry.Name, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -19,13 +19,13 @@ type OneDriveStorage struct {
|
||||
}
|
||||
|
||||
// CreateOneDriveStorage creates an OneDrive storage object.
|
||||
func CreateOneDriveStorage(tokenFile string, isBusiness bool, storagePath string, threads int) (storage *OneDriveStorage, err error) {
|
||||
func CreateOneDriveStorage(tokenFile string, isBusiness bool, storagePath string, threads int, client_id string, client_secret string, drive_id string) (storage *OneDriveStorage, err error) {
|
||||
|
||||
for len(storagePath) > 0 && storagePath[len(storagePath)-1] == '/' {
|
||||
storagePath = storagePath[:len(storagePath)-1]
|
||||
}
|
||||
|
||||
client, err := NewOneDriveClient(tokenFile, isBusiness)
|
||||
client, err := NewOneDriveClient(tokenFile, isBusiness, client_id, client_secret, drive_id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -6,27 +6,57 @@ package duplicacy
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type flagsMask uint32
|
||||
|
||||
func (f flagsMask) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(fmt.Sprintf("0x%.8x", f))
|
||||
}
|
||||
|
||||
func (f *flagsMask) UnmarshalJSON(data []byte) error {
|
||||
var str string
|
||||
if err := json.Unmarshal(data, &str); err != nil {
|
||||
return err
|
||||
}
|
||||
if str[0] == '0' && (str[1] == 'x' || str[1] == 'X') {
|
||||
str = str[2:]
|
||||
}
|
||||
|
||||
v, err := strconv.ParseUint(string(str), 16, 32)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*f = flagsMask(v)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Preference stores options for each storage.
|
||||
type Preference struct {
|
||||
Name string `json:"name"`
|
||||
SnapshotID string `json:"id"`
|
||||
RepositoryPath string `json:"repository"`
|
||||
StorageURL string `json:"storage"`
|
||||
Encrypted bool `json:"encrypted"`
|
||||
BackupProhibited bool `json:"no_backup"`
|
||||
RestoreProhibited bool `json:"no_restore"`
|
||||
DoNotSavePassword bool `json:"no_save_password"`
|
||||
NobackupFile string `json:"nobackup_file"`
|
||||
Keys map[string]string `json:"keys"`
|
||||
FiltersFile string `json:"filters"`
|
||||
ExcludeByAttribute bool `json:"exclude_by_attribute"`
|
||||
Name string `json:"name"`
|
||||
SnapshotID string `json:"id"`
|
||||
RepositoryPath string `json:"repository"`
|
||||
StorageURL string `json:"storage"`
|
||||
Encrypted bool `json:"encrypted"`
|
||||
BackupProhibited bool `json:"no_backup"`
|
||||
RestoreProhibited bool `json:"no_restore"`
|
||||
DoNotSavePassword bool `json:"no_save_password"`
|
||||
NobackupFile string `json:"nobackup_file"`
|
||||
Keys map[string]string `json:"keys"`
|
||||
FiltersFile string `json:"filters"`
|
||||
ExcludeOwner bool `json:"exclude_owner"`
|
||||
ExcludeByAttribute bool `json:"exclude_by_attribute"`
|
||||
ExcludeXattrs bool `json:"exclude_xattrs"`
|
||||
NormalizeXattrs bool `json:"normalize_xattrs"`
|
||||
IncludeFileFlags bool `json:"include_file_flags"`
|
||||
IncludeSpecials bool `json:"include_specials"`
|
||||
FileFlagsMask flagsMask `json:"file_flags_mask"`
|
||||
}
|
||||
|
||||
var preferencePath string
|
||||
@@ -43,7 +73,7 @@ func LoadPreferences(repository string) bool {
|
||||
}
|
||||
|
||||
if !stat.IsDir() {
|
||||
content, err := ioutil.ReadFile(preferencePath)
|
||||
content, err := os.ReadFile(preferencePath)
|
||||
if err != nil {
|
||||
LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to locate the preference path: %v", err)
|
||||
return false
|
||||
@@ -61,7 +91,7 @@ func LoadPreferences(repository string) bool {
|
||||
preferencePath = realPreferencePath
|
||||
}
|
||||
|
||||
description, err := ioutil.ReadFile(path.Join(preferencePath, "preferences"))
|
||||
description, err := os.ReadFile(path.Join(preferencePath, "preferences"))
|
||||
if err != nil {
|
||||
LOG_ERROR("PREFERENCE_OPEN", "Failed to read the preference file from repository %s: %v", repository, err)
|
||||
return false
|
||||
@@ -110,7 +140,7 @@ func SavePreferences() bool {
|
||||
}
|
||||
preferenceFile := path.Join(GetDuplicacyPreferencePath(), "preferences")
|
||||
|
||||
err = ioutil.WriteFile(preferenceFile, description, 0600)
|
||||
err = os.WriteFile(preferenceFile, description, 0600)
|
||||
if err != nil {
|
||||
LOG_ERROR("PREFERENCE_WRITE", "Failed to save the preference file %s: %v", preferenceFile, err)
|
||||
return false
|
||||
|
||||
@@ -90,48 +90,40 @@ func (storage *S3Storage) ListFiles(threadIndex int, dir string) (files []string
|
||||
|
||||
if dir == "snapshots/" {
|
||||
dir = storage.storageDir + dir
|
||||
input := s3.ListObjectsInput{
|
||||
input := s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(storage.bucket),
|
||||
Prefix: aws.String(dir),
|
||||
Delimiter: aws.String("/"),
|
||||
MaxKeys: aws.Int64(1000),
|
||||
}
|
||||
|
||||
output, err := storage.client.ListObjects(&input)
|
||||
err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
|
||||
for _, subDir := range page.CommonPrefixes {
|
||||
files = append(files, (*subDir.Prefix)[len(dir):])
|
||||
}
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
for _, subDir := range output.CommonPrefixes {
|
||||
files = append(files, (*subDir.Prefix)[len(dir):])
|
||||
}
|
||||
return files, nil, nil
|
||||
} else {
|
||||
dir = storage.storageDir + dir
|
||||
marker := ""
|
||||
for {
|
||||
input := s3.ListObjectsInput{
|
||||
Bucket: aws.String(storage.bucket),
|
||||
Prefix: aws.String(dir),
|
||||
MaxKeys: aws.Int64(1000),
|
||||
Marker: aws.String(marker),
|
||||
}
|
||||
input := s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(storage.bucket),
|
||||
Prefix: aws.String(dir),
|
||||
MaxKeys: aws.Int64(1000),
|
||||
}
|
||||
|
||||
output, err := storage.client.ListObjects(&input)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
for _, object := range output.Contents {
|
||||
err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
|
||||
for _, object := range page.Contents {
|
||||
files = append(files, (*object.Key)[len(dir):])
|
||||
sizes = append(sizes, *object.Size)
|
||||
}
|
||||
|
||||
if !*output.IsTruncated {
|
||||
break
|
||||
}
|
||||
|
||||
marker = *output.Contents[len(output.Contents)-1].Key
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return files, sizes, nil
|
||||
}
|
||||
|
||||
250
src/duplicacy_sambastorage.go
Normal file
250
src/duplicacy_sambastorage.go
Normal file
@@ -0,0 +1,250 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"fmt"
|
||||
"net"
|
||||
"path"
|
||||
"time"
|
||||
"strings"
|
||||
"syscall"
|
||||
"math/rand"
|
||||
|
||||
"github.com/hirochachacha/go-smb2"
|
||||
)
|
||||
|
||||
// SambaStorage is a local on-disk file storage implementing the Storage interface.
|
||||
type SambaStorage struct {
|
||||
StorageBase
|
||||
|
||||
share *smb2.Share
|
||||
storageDir string
|
||||
numberOfThreads int
|
||||
}
|
||||
|
||||
// CreateSambaStorage creates a file storage.
|
||||
func CreateSambaStorage(server string, port int, username string, password string, shareName string, storageDir string, threads int) (storage *SambaStorage, err error) {
|
||||
|
||||
connection, err := net.Dial("tcp", fmt.Sprintf("%s:%d", server, port))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dialer := &smb2.Dialer{
|
||||
Initiator: &smb2.NTLMInitiator{
|
||||
User: username,
|
||||
Password: password,
|
||||
},
|
||||
}
|
||||
|
||||
client, err := dialer.Dial(connection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
share, err := client.Mount(shareName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Random number fo generating the temporary chunk file suffix.
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
|
||||
storage = &SambaStorage{
|
||||
share: share,
|
||||
numberOfThreads: threads,
|
||||
}
|
||||
|
||||
exist, isDir, _, err := storage.GetFileInfo(0, storageDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to check the storage path %s: %v", storageDir, err)
|
||||
}
|
||||
|
||||
if !exist {
|
||||
return nil, fmt.Errorf("The storage path %s does not exist", storageDir)
|
||||
}
|
||||
|
||||
if !isDir {
|
||||
return nil, fmt.Errorf("The storage path %s is not a directory", storageDir)
|
||||
}
|
||||
|
||||
storage.storageDir = storageDir
|
||||
storage.DerivedStorage = storage
|
||||
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
// ListFiles return the list of files and subdirectories under 'dir' (non-recursively).
|
||||
func (storage *SambaStorage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) {
|
||||
|
||||
fullPath := path.Join(storage.storageDir, dir)
|
||||
|
||||
list, err := storage.share.ReadDir(fullPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
for _, f := range list {
|
||||
name := f.Name()
|
||||
if (f.IsDir() || f.Mode() & os.ModeSymlink != 0) && name[len(name)-1] != '/' {
|
||||
name += "/"
|
||||
}
|
||||
files = append(files, name)
|
||||
sizes = append(sizes, f.Size())
|
||||
}
|
||||
|
||||
return files, sizes, nil
|
||||
}
|
||||
|
||||
// DeleteFile deletes the file or directory at 'filePath'.
|
||||
func (storage *SambaStorage) DeleteFile(threadIndex int, filePath string) (err error) {
|
||||
err = storage.share.Remove(path.Join(storage.storageDir, filePath))
|
||||
if err == nil || os.IsNotExist(err) {
|
||||
return nil
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// MoveFile renames the file.
|
||||
func (storage *SambaStorage) MoveFile(threadIndex int, from string, to string) (err error) {
|
||||
return storage.share.Rename(path.Join(storage.storageDir, from), path.Join(storage.storageDir, to))
|
||||
}
|
||||
|
||||
// CreateDirectory creates a new directory.
|
||||
func (storage *SambaStorage) CreateDirectory(threadIndex int, dir string) (err error) {
|
||||
fmt.Printf("Creating directory %s\n", dir)
|
||||
err = storage.share.Mkdir(path.Join(storage.storageDir, dir), 0744)
|
||||
if err != nil && os.IsExist(err) {
|
||||
return nil
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// GetFileInfo returns the information about the file or directory at 'filePath'.
|
||||
func (storage *SambaStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
|
||||
stat, err := storage.share.Stat(path.Join(storage.storageDir, filePath))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, false, 0, nil
|
||||
} else {
|
||||
return false, false, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
return true, stat.IsDir(), stat.Size(), nil
|
||||
}
|
||||
|
||||
// DownloadFile reads the file at 'filePath' into the chunk.
|
||||
func (storage *SambaStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
|
||||
|
||||
file, err := storage.share.Open(path.Join(storage.storageDir, filePath))
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer file.Close()
|
||||
if _, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit/storage.numberOfThreads); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// UploadFile writes 'content' to the file at 'filePath'
|
||||
func (storage *SambaStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
|
||||
|
||||
fullPath := path.Join(storage.storageDir, filePath)
|
||||
|
||||
if len(strings.Split(filePath, "/")) > 2 {
|
||||
dir := path.Dir(fullPath)
|
||||
stat, err := storage.share.Stat(dir)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
err = storage.share.MkdirAll(dir, 0744)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if !stat.IsDir() && stat.Mode() & os.ModeSymlink == 0 {
|
||||
return fmt.Errorf("The path %s is not a directory or symlink", dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
letters := "abcdefghijklmnopqrstuvwxyz"
|
||||
suffix := make([]byte, 8)
|
||||
for i := range suffix {
|
||||
suffix[i] = letters[rand.Intn(len(letters))]
|
||||
}
|
||||
|
||||
temporaryFile := fullPath + "." + string(suffix) + ".tmp"
|
||||
|
||||
file, err := storage.share.Create(temporaryFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
|
||||
_, err = io.Copy(file, reader)
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
if err = file.Sync(); err != nil {
|
||||
pathErr, ok := err.(*os.PathError)
|
||||
isNotSupported := ok && pathErr.Op == "sync" && pathErr.Err == syscall.ENOTSUP
|
||||
if !isNotSupported {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = file.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = storage.share.Rename(temporaryFile, fullPath)
|
||||
if err != nil {
|
||||
|
||||
if _, e := storage.share.Stat(fullPath); e == nil {
|
||||
storage.share.Remove(temporaryFile)
|
||||
return nil
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
|
||||
// managing snapshots.
|
||||
func (storage *SambaStorage) IsCacheNeeded() bool { return true }
|
||||
|
||||
// If the 'MoveFile' method is implemented.
|
||||
func (storage *SambaStorage) IsMoveFileImplemented() bool { return true }
|
||||
|
||||
// If the storage can guarantee strong consistency.
|
||||
func (storage *SambaStorage) IsStrongConsistent() bool { return true }
|
||||
|
||||
// If the storage supports fast listing of files names.
|
||||
func (storage *SambaStorage) IsFastListing() bool { return false }
|
||||
|
||||
// Enable the test mode.
|
||||
func (storage *SambaStorage) EnableTestMode() {}
|
||||
@@ -144,14 +144,14 @@ func (storage *SFTPStorage) retry(f func () error) error {
|
||||
storage.clientLock.Lock()
|
||||
connection, err := ssh.Dial("tcp", storage.serverAddress, storage.sftpConfig)
|
||||
if err != nil {
|
||||
LOG_WARN("SFT_RECONNECT", "Failed to connect to %s: %v; retrying", storage.serverAddress, err)
|
||||
LOG_WARN("SFTP_RECONNECT", "Failed to connect to %s: %v; retrying", storage.serverAddress, err)
|
||||
storage.clientLock.Unlock()
|
||||
continue
|
||||
}
|
||||
|
||||
client, err := sftp.NewClient(connection)
|
||||
if err != nil {
|
||||
LOG_WARN("SFT_RECONNECT", "Failed to create a new SFTP client to %s: %v; retrying", storage.serverAddress, err)
|
||||
LOG_WARN("SFTP_RECONNECT", "Failed to create a new SFTP client to %s: %v; retrying", storage.serverAddress, err)
|
||||
connection.Close()
|
||||
storage.clientLock.Unlock()
|
||||
continue
|
||||
|
||||
@@ -10,7 +10,6 @@ package duplicacy
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
@@ -136,7 +135,7 @@ func CreateShadowCopy(top string, shadowCopy bool, timeoutInSeconds int) (shadow
|
||||
}
|
||||
|
||||
// Create mount point
|
||||
snapshotPath, err = ioutil.TempDir("/tmp/", "snp_")
|
||||
snapshotPath, err = os.MkdirTemp("/tmp/", "snp_")
|
||||
if err != nil {
|
||||
LOG_ERROR("VSS_CREATE", "Failed to create temporary mount directory")
|
||||
return top
|
||||
|
||||
@@ -9,16 +9,13 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"sort"
|
||||
"bytes"
|
||||
|
||||
"github.com/vmihailenco/msgpack"
|
||||
|
||||
"github.com/vmihailenco/msgpack"
|
||||
)
|
||||
|
||||
// Snapshot represents a backup of the repository.
|
||||
@@ -52,6 +49,7 @@ type Snapshot struct {
|
||||
// CreateEmptySnapshot creates an empty snapshot.
|
||||
func CreateEmptySnapshot(id string) (snapshto *Snapshot) {
|
||||
return &Snapshot{
|
||||
Version: 1,
|
||||
ID: id,
|
||||
Revision: 0,
|
||||
StartTime: time.Now().Unix(),
|
||||
@@ -60,19 +58,41 @@ func CreateEmptySnapshot(id string) (snapshto *Snapshot) {
|
||||
|
||||
type DirectoryListing struct {
|
||||
directory string
|
||||
files *[]Entry
|
||||
files *[]Entry
|
||||
}
|
||||
|
||||
func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
|
||||
filtersFile string, excludeByAttribute bool, listingChannel chan *Entry,
|
||||
skippedDirectories *[]string, skippedFiles *[]string) {
|
||||
type ListFilesOptions struct {
|
||||
NoBackupFile string
|
||||
FiltersFile string
|
||||
ExcludeByAttribute bool
|
||||
ExcludeXattrs bool
|
||||
NormalizeXattr bool
|
||||
IncludeFileFlags bool
|
||||
IncludeSpecials bool
|
||||
}
|
||||
|
||||
var patterns []string
|
||||
|
||||
if filtersFile == "" {
|
||||
filtersFile = joinPath(GetDuplicacyPreferencePath(), "filters")
|
||||
func NewListFilesOptions(p *Preference) *ListFilesOptions {
|
||||
return &ListFilesOptions{
|
||||
NoBackupFile: p.NobackupFile,
|
||||
FiltersFile: p.FiltersFile,
|
||||
ExcludeByAttribute: p.ExcludeByAttribute,
|
||||
ExcludeXattrs: p.ExcludeXattrs,
|
||||
NormalizeXattr: p.NormalizeXattrs,
|
||||
IncludeFileFlags: p.IncludeFileFlags,
|
||||
IncludeSpecials: p.IncludeSpecials,
|
||||
}
|
||||
patterns = ProcessFilters(filtersFile)
|
||||
}
|
||||
|
||||
func (snapshot *Snapshot) ListLocalFiles(top string,
|
||||
listingChannel chan *Entry, skippedDirectories *[]string, skippedFiles *[]string,
|
||||
options *ListFilesOptions) {
|
||||
|
||||
if options.FiltersFile == "" {
|
||||
options.FiltersFile = joinPath(GetDuplicacyPreferencePath(), "filters")
|
||||
}
|
||||
|
||||
patterns := ProcessFilters(options.FiltersFile)
|
||||
lister := NewLocalDirectoryLister()
|
||||
|
||||
directories := make([]*Entry, 0, 256)
|
||||
directories = append(directories, CreateEntry("", 0, 0, 0))
|
||||
@@ -81,7 +101,16 @@ func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
|
||||
|
||||
directory := directories[len(directories)-1]
|
||||
directories = directories[:len(directories)-1]
|
||||
subdirectories, skipped, err := ListEntries(top, directory.Path, patterns, nobackupFile, excludeByAttribute, listingChannel)
|
||||
subdirectories, skipped, err := lister.ListDir(top, directory.Path, listingChannel,
|
||||
&EntryListerOptions{
|
||||
Patterns: patterns,
|
||||
NoBackupFile: options.NoBackupFile,
|
||||
ExcludeByAttribute: options.ExcludeByAttribute,
|
||||
ExcludeXattrs: options.ExcludeXattrs,
|
||||
NormalizeXattr: options.NormalizeXattr,
|
||||
IncludeFileFlags: options.IncludeFileFlags,
|
||||
IncludeSpecials: options.IncludeSpecials,
|
||||
})
|
||||
if err != nil {
|
||||
if directory.Path == "" {
|
||||
LOG_ERROR("LIST_FAILURE", "Failed to list the repository root: %v", err)
|
||||
@@ -104,7 +133,7 @@ func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
|
||||
close(listingChannel)
|
||||
}
|
||||
|
||||
func (snapshot *Snapshot)ListRemoteFiles(config *Config, chunkOperator *ChunkOperator, entryOut func(*Entry) bool) {
|
||||
func (snapshot *Snapshot) ListRemoteFiles(config *Config, chunkOperator *ChunkOperator, entryOut func(*Entry) bool) {
|
||||
|
||||
var chunks []string
|
||||
for _, chunkHash := range snapshot.FileSequence {
|
||||
@@ -112,22 +141,27 @@ func (snapshot *Snapshot)ListRemoteFiles(config *Config, chunkOperator *ChunkOpe
|
||||
}
|
||||
|
||||
var chunk *Chunk
|
||||
reader := sequenceReader{
|
||||
sequence: snapshot.FileSequence,
|
||||
buffer: new(bytes.Buffer),
|
||||
refillFunc: func(chunkHash string) []byte {
|
||||
if chunk != nil {
|
||||
config.PutChunk(chunk)
|
||||
}
|
||||
chunk = chunkOperator.Download(chunkHash, 0, true)
|
||||
return chunk.GetBytes()
|
||||
},
|
||||
}
|
||||
reader := NewSequenceReader(snapshot.FileSequence, func(chunkHash string) []byte {
|
||||
if chunk != nil {
|
||||
config.PutChunk(chunk)
|
||||
}
|
||||
chunk = chunkOperator.Download(chunkHash, 0, true)
|
||||
return chunk.GetBytes()
|
||||
})
|
||||
|
||||
if snapshot.Version == 0 {
|
||||
defer func() {
|
||||
if chunk != nil {
|
||||
config.PutChunk(chunk)
|
||||
}
|
||||
}()
|
||||
|
||||
// Normally if Version is 0 then the snapshot is created by CLI v2 but unfortunately CLI 3.0.1 does not set the
|
||||
// version bit correctly when copying old backups. So we need to check the first byte -- if it is '[' then it is
|
||||
// the old format. The new format starts with a string encoded in msgpack and the first byte can't be '['.
|
||||
if snapshot.Version == 0 || reader.GetFirstByte() == '[' {
|
||||
LOG_INFO("SNAPSHOT_VERSION", "snapshot %s at revision %d is encoded in an old version format", snapshot.ID, snapshot.Revision)
|
||||
files := make([]*Entry, 0)
|
||||
decoder := json.NewDecoder(&reader)
|
||||
decoder := json.NewDecoder(reader)
|
||||
|
||||
// read open bracket
|
||||
_, err := decoder.Token()
|
||||
@@ -156,7 +190,7 @@ func (snapshot *Snapshot)ListRemoteFiles(config *Config, chunkOperator *ChunkOpe
|
||||
}
|
||||
}
|
||||
} else if snapshot.Version == 1 {
|
||||
decoder := msgpack.NewDecoder(&reader)
|
||||
decoder := msgpack.NewDecoder(reader)
|
||||
|
||||
lastEndChunk := 0
|
||||
|
||||
@@ -195,7 +229,7 @@ func (snapshot *Snapshot)ListRemoteFiles(config *Config, chunkOperator *ChunkOpe
|
||||
|
||||
} else {
|
||||
LOG_ERROR("SNAPSHOT_VERSION", "snapshot %s at revision %d is encoded in unsupported version %d format",
|
||||
snapshot.ID, snapshot.Revision, snapshot.Version)
|
||||
snapshot.ID, snapshot.Revision, snapshot.Version)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -238,7 +272,7 @@ func ProcessFilterFile(patternFile string, includedFiles []string) (patterns []s
|
||||
}
|
||||
includedFiles = append(includedFiles, patternFile)
|
||||
LOG_INFO("SNAPSHOT_FILTER", "Parsing filter file %s", patternFile)
|
||||
patternFileContent, err := ioutil.ReadFile(patternFile)
|
||||
patternFileContent, err := os.ReadFile(patternFile)
|
||||
if err == nil {
|
||||
patternFileLines := strings.Split(string(patternFileContent), "\n")
|
||||
patterns = ProcessFilterLines(patternFileLines, includedFiles)
|
||||
@@ -258,7 +292,7 @@ func ProcessFilterLines(patternFileLines []string, includedFiles []string) (patt
|
||||
if patternIncludeFile == "" {
|
||||
continue
|
||||
}
|
||||
if ! filepath.IsAbs(patternIncludeFile) {
|
||||
if !filepath.IsAbs(patternIncludeFile) {
|
||||
basePath := ""
|
||||
if len(includedFiles) == 0 {
|
||||
basePath, _ = os.Getwd()
|
||||
@@ -434,7 +468,7 @@ func (snapshot *Snapshot) MarshalJSON() ([]byte, error) {
|
||||
|
||||
object := make(map[string]interface{})
|
||||
|
||||
object["version"] = 1
|
||||
object["version"] = snapshot.Version
|
||||
object["id"] = snapshot.ID
|
||||
object["revision"] = snapshot.Revision
|
||||
object["options"] = snapshot.Options
|
||||
@@ -485,4 +519,3 @@ func encodeSequence(sequence []string) []string {
|
||||
|
||||
return sequenceInHex
|
||||
}
|
||||
|
||||
|
||||
@@ -18,10 +18,10 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/aryann/difflib"
|
||||
)
|
||||
@@ -191,7 +191,7 @@ type SnapshotManager struct {
|
||||
fileChunk *Chunk
|
||||
snapshotCache *FileStorage
|
||||
|
||||
chunkOperator *ChunkOperator
|
||||
chunkOperator *ChunkOperator
|
||||
}
|
||||
|
||||
// CreateSnapshotManager creates a snapshot manager
|
||||
@@ -249,17 +249,27 @@ func (manager *SnapshotManager) DownloadSnapshot(snapshotID string, revision int
|
||||
// the memory before passing them to the json unmarshaller.
|
||||
type sequenceReader struct {
|
||||
sequence []string
|
||||
buffer *bytes.Buffer
|
||||
buffer *bytes.Reader
|
||||
index int
|
||||
refillFunc func(hash string) []byte
|
||||
}
|
||||
|
||||
func NewSequenceReader(sequence []string, refillFunc func(hash string) []byte) *sequenceReader {
|
||||
newData := refillFunc(sequence[0])
|
||||
return &sequenceReader{
|
||||
sequence: sequence,
|
||||
buffer: bytes.NewReader(newData),
|
||||
index: 1,
|
||||
refillFunc: refillFunc,
|
||||
}
|
||||
}
|
||||
|
||||
// Read reads a new chunk using the refill function when there is no more data in the buffer
|
||||
func (reader *sequenceReader) Read(data []byte) (n int, err error) {
|
||||
if len(reader.buffer.Bytes()) == 0 {
|
||||
if reader.buffer.Len() == 0 {
|
||||
if reader.index < len(reader.sequence) {
|
||||
newData := reader.refillFunc(reader.sequence[reader.index])
|
||||
reader.buffer.Write(newData)
|
||||
reader.buffer = bytes.NewReader(newData)
|
||||
reader.index++
|
||||
} else {
|
||||
return 0, io.EOF
|
||||
@@ -269,15 +279,25 @@ func (reader *sequenceReader) Read(data []byte) (n int, err error) {
|
||||
return reader.buffer.Read(data)
|
||||
}
|
||||
|
||||
func (manager *SnapshotManager) CreateChunkOperator(resurrect bool, threads int, allowFailures bool) {
|
||||
func (reader *sequenceReader) GetFirstByte() byte {
|
||||
b, err := reader.buffer.ReadByte()
|
||||
reader.buffer.UnreadByte()
|
||||
if err != nil {
|
||||
return 0
|
||||
} else {
|
||||
return b
|
||||
}
|
||||
}
|
||||
|
||||
func (manager *SnapshotManager) CreateChunkOperator(resurrect bool, rewriteChunks bool, threads int, allowFailures bool) {
|
||||
if manager.chunkOperator == nil {
|
||||
manager.chunkOperator = CreateChunkOperator(manager.config, manager.storage, manager.snapshotCache, resurrect, threads, allowFailures)
|
||||
manager.chunkOperator = CreateChunkOperator(manager.config, manager.storage, manager.snapshotCache, resurrect, rewriteChunks, threads, allowFailures)
|
||||
}
|
||||
}
|
||||
|
||||
// DownloadSequence returns the content represented by a sequence of chunks.
|
||||
func (manager *SnapshotManager) DownloadSequence(sequence []string) (content []byte) {
|
||||
manager.CreateChunkOperator(false, 1, false)
|
||||
manager.CreateChunkOperator(false, false, 1, false)
|
||||
for _, chunkHash := range sequence {
|
||||
chunk := manager.chunkOperator.Download(chunkHash, 0, true)
|
||||
content = append(content, chunk.GetBytes()...)
|
||||
@@ -428,7 +448,7 @@ func (manager *SnapshotManager) CleanSnapshotCache(latestSnapshot *Snapshot, all
|
||||
|
||||
allFiles, _ := manager.ListAllFiles(manager.snapshotCache, chunkDir)
|
||||
for _, file := range allFiles {
|
||||
if file[len(file)-1] != '/' {
|
||||
if len(file) > 0 && file[len(file)-1] != '/' {
|
||||
chunkID := strings.Replace(file, "/", "", -1)
|
||||
if _, found := chunks[chunkID]; !found {
|
||||
LOG_DEBUG("SNAPSHOT_CLEAN", "Delete chunk %s from the snapshot cache", chunkID)
|
||||
@@ -654,7 +674,7 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
|
||||
LOG_DEBUG("LIST_PARAMETERS", "id: %s, revisions: %v, tag: %s, showFiles: %t, showChunks: %t",
|
||||
snapshotID, revisionsToList, tag, showFiles, showChunks)
|
||||
|
||||
manager.CreateChunkOperator(false, 1, false)
|
||||
manager.CreateChunkOperator(false, false, 1, false)
|
||||
defer func() {
|
||||
manager.chunkOperator.Stop()
|
||||
manager.chunkOperator = nil
|
||||
@@ -718,7 +738,7 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
|
||||
totalFileSize := int64(0)
|
||||
lastChunk := 0
|
||||
|
||||
snapshot.ListRemoteFiles(manager.config, manager.chunkOperator, func(file *Entry)bool {
|
||||
snapshot.ListRemoteFiles(manager.config, manager.chunkOperator, func(file *Entry) bool {
|
||||
if file.IsFile() {
|
||||
totalFiles++
|
||||
totalFileSize += file.Size
|
||||
@@ -733,7 +753,7 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
|
||||
return true
|
||||
})
|
||||
|
||||
snapshot.ListRemoteFiles(manager.config, manager.chunkOperator, func(file *Entry)bool {
|
||||
snapshot.ListRemoteFiles(manager.config, manager.chunkOperator, func(file *Entry) bool {
|
||||
if file.IsFile() {
|
||||
LOG_INFO("SNAPSHOT_FILE", "%s", file.String(maxSizeDigits))
|
||||
}
|
||||
@@ -760,9 +780,9 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
|
||||
|
||||
// CheckSnapshots checks if there is any problem with a snapshot.
|
||||
func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToCheck []int, tag string, showStatistics bool, showTabular bool,
|
||||
checkFiles bool, checkChunks, searchFossils bool, resurrect bool, threads int, allowFailures bool) bool {
|
||||
checkFiles bool, checkChunks, searchFossils bool, resurrect bool, rewriteChunks bool, threads int, allowFailures bool) bool {
|
||||
|
||||
manager.CreateChunkOperator(resurrect, threads, allowFailures)
|
||||
manager.CreateChunkOperator(resurrect, rewriteChunks, threads, allowFailures)
|
||||
defer func() {
|
||||
manager.chunkOperator.Stop()
|
||||
manager.chunkOperator = nil
|
||||
@@ -888,7 +908,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
|
||||
_, exist, _, err := manager.storage.FindChunk(0, chunkID, false)
|
||||
if err != nil {
|
||||
LOG_WARN("SNAPSHOT_VALIDATE", "Failed to check the existence of chunk %s: %v",
|
||||
chunkID, err)
|
||||
chunkID, err)
|
||||
} else if exist {
|
||||
LOG_INFO("SNAPSHOT_VALIDATE", "Chunk %s is confirmed to exist", chunkID)
|
||||
continue
|
||||
@@ -999,6 +1019,8 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
|
||||
numberOfVerifiedChunks := len(verifiedChunks)
|
||||
|
||||
saveVerifiedChunks := func() {
|
||||
verifiedChunksLock.Lock()
|
||||
defer verifiedChunksLock.Unlock()
|
||||
if len(verifiedChunks) > numberOfVerifiedChunks {
|
||||
var description []byte
|
||||
description, err = json.Marshal(verifiedChunks)
|
||||
@@ -1009,7 +1031,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
|
||||
if err != nil {
|
||||
LOG_WARN("SNAPSHOT_VERIFY", "Failed to save the verified chunks file: %v", err)
|
||||
} else {
|
||||
LOG_INFO("SNAPSHOT_VERIFY", "Added %d chunks to the list of verified chunks", len(verifiedChunks) - numberOfVerifiedChunks)
|
||||
LOG_INFO("SNAPSHOT_VERIFY", "Added %d chunks to the list of verified chunks", len(verifiedChunks)-numberOfVerifiedChunks)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1051,7 +1073,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
|
||||
defer CatchLogException()
|
||||
|
||||
for {
|
||||
chunkIndex, ok := <- chunkChannel
|
||||
chunkIndex, ok := <-chunkChannel
|
||||
if !ok {
|
||||
wg.Done()
|
||||
return
|
||||
@@ -1071,14 +1093,14 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
|
||||
|
||||
elapsedTime := time.Now().Sub(startTime).Seconds()
|
||||
speed := int64(float64(downloadedChunkSize) / elapsedTime)
|
||||
remainingTime := int64(float64(totalChunks - downloadedChunks) / float64(downloadedChunks) * elapsedTime)
|
||||
remainingTime := int64(float64(totalChunks-downloadedChunks) / float64(downloadedChunks) * elapsedTime)
|
||||
percentage := float64(downloadedChunks) / float64(totalChunks) * 100.0
|
||||
LOG_INFO("VERIFY_PROGRESS", "Verified chunk %s (%d/%d), %sB/s %s %.1f%%",
|
||||
chunkID, downloadedChunks, totalChunks, PrettySize(speed), PrettyTime(remainingTime), percentage)
|
||||
chunkID, downloadedChunks, totalChunks, PrettySize(speed), PrettyTime(remainingTime), percentage)
|
||||
|
||||
manager.config.PutChunk(chunk)
|
||||
}
|
||||
} ()
|
||||
}()
|
||||
}
|
||||
|
||||
for chunkIndex := range chunkHashes {
|
||||
@@ -1267,10 +1289,10 @@ func (manager *SnapshotManager) PrintSnapshot(snapshot *Snapshot) bool {
|
||||
}
|
||||
|
||||
// Don't print the ending bracket
|
||||
fmt.Printf("%s", string(description[:len(description) - 2]))
|
||||
fmt.Printf("%s", string(description[:len(description)-2]))
|
||||
fmt.Printf(",\n \"files\": [\n")
|
||||
isFirstFile := true
|
||||
snapshot.ListRemoteFiles(manager.config, manager.chunkOperator, func (file *Entry) bool {
|
||||
snapshot.ListRemoteFiles(manager.config, manager.chunkOperator, func(file *Entry) bool {
|
||||
|
||||
fileDescription, _ := json.MarshalIndent(file.convertToObject(false), "", " ")
|
||||
|
||||
@@ -1300,7 +1322,7 @@ func (manager *SnapshotManager) VerifySnapshot(snapshot *Snapshot) bool {
|
||||
}
|
||||
|
||||
files := make([]*Entry, 0)
|
||||
snapshot.ListRemoteFiles(manager.config, manager.chunkOperator, func (file *Entry) bool {
|
||||
snapshot.ListRemoteFiles(manager.config, manager.chunkOperator, func(file *Entry) bool {
|
||||
if file.IsFile() && file.Size != 0 {
|
||||
file.Attributes = nil
|
||||
files = append(files, file)
|
||||
@@ -1318,6 +1340,10 @@ func (manager *SnapshotManager) VerifySnapshot(snapshot *Snapshot) bool {
|
||||
LOG_TRACE("SNAPSHOT_VERIFY", "%s", file.Path)
|
||||
}
|
||||
|
||||
if lastChunk != nil {
|
||||
manager.config.PutChunk(lastChunk)
|
||||
}
|
||||
|
||||
if corruptedFiles > 0 {
|
||||
LOG_WARN("SNAPSHOT_VERIFY", "Snapshot %s at revision %d contains %d corrupted files",
|
||||
snapshot.ID, snapshot.Revision, corruptedFiles)
|
||||
@@ -1336,7 +1362,7 @@ func (manager *SnapshotManager) RetrieveFile(snapshot *Snapshot, file *Entry, la
|
||||
return true
|
||||
}
|
||||
|
||||
manager.CreateChunkOperator(false, 1, false)
|
||||
manager.CreateChunkOperator(false, false, 1, false)
|
||||
|
||||
fileHasher := manager.config.NewFileHasher()
|
||||
alternateHash := false
|
||||
@@ -1372,6 +1398,11 @@ func (manager *SnapshotManager) RetrieveFile(snapshot *Snapshot, file *Entry, la
|
||||
}
|
||||
}
|
||||
|
||||
if chunk.isBroken {
|
||||
*lastChunk = nil
|
||||
return false
|
||||
}
|
||||
|
||||
output(chunk.GetBytes()[start:end])
|
||||
if alternateHash {
|
||||
fileHasher.Write([]byte(hex.EncodeToString([]byte(hash))))
|
||||
@@ -1395,7 +1426,7 @@ func (manager *SnapshotManager) RetrieveFile(snapshot *Snapshot, file *Entry, la
|
||||
func (manager *SnapshotManager) FindFile(snapshot *Snapshot, filePath string, suppressError bool) *Entry {
|
||||
|
||||
var found *Entry
|
||||
snapshot.ListRemoteFiles(manager.config, manager.chunkOperator, func (entry *Entry) bool {
|
||||
snapshot.ListRemoteFiles(manager.config, manager.chunkOperator, func(entry *Entry) bool {
|
||||
if entry.Path == filePath {
|
||||
found = entry
|
||||
return false
|
||||
@@ -1448,8 +1479,8 @@ func (manager *SnapshotManager) PrintFile(snapshotID string, revision int, path
|
||||
|
||||
file := manager.FindFile(snapshot, path, false)
|
||||
if !manager.RetrieveFile(snapshot, file, nil, func(chunk []byte) {
|
||||
fmt.Printf("%s", chunk)
|
||||
}) {
|
||||
fmt.Printf("%s", chunk)
|
||||
}) {
|
||||
LOG_ERROR("SNAPSHOT_RETRIEVE", "File %s is corrupted in snapshot %s at revision %d",
|
||||
path, snapshot.ID, snapshot.Revision)
|
||||
return false
|
||||
@@ -1460,16 +1491,17 @@ func (manager *SnapshotManager) PrintFile(snapshotID string, revision int, path
|
||||
|
||||
// Diff compares two snapshots, or two revision of a file if the file argument is given.
|
||||
func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []int,
|
||||
filePath string, compareByHash bool, nobackupFile string, filtersFile string, excludeByAttribute bool) bool {
|
||||
filePath string, compareByHash bool,
|
||||
options *ListFilesOptions) bool {
|
||||
|
||||
LOG_DEBUG("DIFF_PARAMETERS", "top: %s, id: %s, revision: %v, path: %s, compareByHash: %t",
|
||||
top, snapshotID, revisions, filePath, compareByHash)
|
||||
|
||||
manager.CreateChunkOperator(false, 1, false)
|
||||
manager.CreateChunkOperator(false, false, 1, false)
|
||||
defer func() {
|
||||
manager.chunkOperator.Stop()
|
||||
manager.chunkOperator = nil
|
||||
} ()
|
||||
}()
|
||||
|
||||
var leftSnapshot *Snapshot
|
||||
var rightSnapshot *Snapshot
|
||||
@@ -1485,11 +1517,11 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []
|
||||
localListingChannel := make(chan *Entry)
|
||||
go func() {
|
||||
defer CatchLogException()
|
||||
rightSnapshot.ListLocalFiles(top, nobackupFile, filtersFile, excludeByAttribute, localListingChannel, nil, nil)
|
||||
} ()
|
||||
rightSnapshot.ListLocalFiles(top, localListingChannel, nil, nil, options)
|
||||
}()
|
||||
|
||||
for entry := range localListingChannel {
|
||||
entry.Attributes = nil // attributes are not compared
|
||||
entry.Attributes = nil // attributes are not compared
|
||||
rightSnapshotFiles = append(rightSnapshotFiles, entry)
|
||||
}
|
||||
|
||||
@@ -1690,11 +1722,11 @@ func (manager *SnapshotManager) ShowHistory(top string, snapshotID string, revis
|
||||
LOG_DEBUG("HISTORY_PARAMETERS", "top: %s, id: %s, revisions: %v, path: %s, showLocalHash: %t",
|
||||
top, snapshotID, revisions, filePath, showLocalHash)
|
||||
|
||||
manager.CreateChunkOperator(false, 1, false)
|
||||
manager.CreateChunkOperator(false, false, 1, false)
|
||||
defer func() {
|
||||
manager.chunkOperator.Stop()
|
||||
manager.chunkOperator = nil
|
||||
} ()
|
||||
}()
|
||||
|
||||
var err error
|
||||
|
||||
@@ -1790,15 +1822,16 @@ func (manager *SnapshotManager) resurrectChunk(fossilPath string, chunkID string
|
||||
|
||||
// PruneSnapshots deletes snapshots by revisions, tags, or a retention policy. The main idea is two-step
|
||||
// fossil collection.
|
||||
// 1. Delete snapshots specified by revision, retention policy, with a tag. Find any resulting unreferenced
|
||||
// chunks, and mark them as fossils (by renaming). After that, create a fossil collection file containing
|
||||
// fossils collected during current run, and temporary files encountered. Also in the file is the latest
|
||||
// revision for each snapshot id. Save this file to a local directory.
|
||||
//
|
||||
// 2. On next run, check if there is any new revision for each snapshot. Or if the lastest revision is too
|
||||
// old, for instance, more than 7 days. This step is to identify snapshots that were being created while
|
||||
// step 1 is in progress. For each fossil reference by any of these snapshots, move them back to the
|
||||
// normal chunk directory.
|
||||
// 1. Delete snapshots specified by revision, retention policy, with a tag. Find any resulting unreferenced
|
||||
// chunks, and mark them as fossils (by renaming). After that, create a fossil collection file containing
|
||||
// fossils collected during current run, and temporary files encountered. Also in the file is the latest
|
||||
// revision for each snapshot id. Save this file to a local directory.
|
||||
//
|
||||
// 2. On next run, check if there is any new revision for each snapshot. Or if the lastest revision is too
|
||||
// old, for instance, more than 7 days. This step is to identify snapshots that were being created while
|
||||
// step 1 is in progress. For each fossil reference by any of these snapshots, move them back to the
|
||||
// normal chunk directory.
|
||||
//
|
||||
// Note that a snapshot being created when step 2 is in progress may reference a fossil. To avoid this
|
||||
// problem, never remove the lastest revision (unless exclusive is true), and only cache chunks referenced
|
||||
@@ -1818,11 +1851,11 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
|
||||
LOG_WARN("DELETE_OPTIONS", "Tags or retention policy will be ignored if at least one revision is specified")
|
||||
}
|
||||
|
||||
manager.CreateChunkOperator(false, threads, false)
|
||||
manager.CreateChunkOperator(false, false, threads, false)
|
||||
defer func() {
|
||||
manager.chunkOperator.Stop()
|
||||
manager.chunkOperator = nil
|
||||
} ()
|
||||
}()
|
||||
|
||||
prefPath := GetDuplicacyPreferencePath()
|
||||
logDir := path.Join(prefPath, "logs")
|
||||
@@ -2396,8 +2429,14 @@ func (manager *SnapshotManager) pruneSnapshotsExhaustive(referencedFossils map[s
|
||||
}
|
||||
|
||||
allFiles, _ := manager.ListAllFiles(manager.storage, chunkDir)
|
||||
uniqueFiles := make(map[string]bool)
|
||||
for _, file := range allFiles {
|
||||
if file[len(file)-1] == '/' {
|
||||
if _, found := uniqueFiles[file]; found {
|
||||
continue
|
||||
}
|
||||
uniqueFiles[file] = true
|
||||
|
||||
if len(file) == 0 || file[len(file)-1] == '/' {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -2507,7 +2546,7 @@ func (manager *SnapshotManager) CheckSnapshot(snapshot *Snapshot) (err error) {
|
||||
numberOfChunks, len(snapshot.ChunkLengths))
|
||||
}
|
||||
|
||||
snapshot.ListRemoteFiles(manager.config, manager.chunkOperator, func (entry *Entry) bool {
|
||||
snapshot.ListRemoteFiles(manager.config, manager.chunkOperator, func(entry *Entry) bool {
|
||||
|
||||
if lastEntry != nil && lastEntry.Compare(entry) >= 0 && !strings.Contains(lastEntry.Path, "\ufffd") {
|
||||
err = fmt.Errorf("The entry %s appears before the entry %s", lastEntry.Path, entry.Path)
|
||||
@@ -2531,7 +2570,7 @@ func (manager *SnapshotManager) CheckSnapshot(snapshot *Snapshot) (err error) {
|
||||
}
|
||||
|
||||
if entry.EndChunk < entry.StartChunk {
|
||||
fmt.Errorf("The file %s starts at chunk %d and ends at chunk %d",
|
||||
err = fmt.Errorf("The file %s starts at chunk %d and ends at chunk %d",
|
||||
entry.Path, entry.StartChunk, entry.EndChunk)
|
||||
return false
|
||||
}
|
||||
@@ -2561,7 +2600,7 @@ func (manager *SnapshotManager) CheckSnapshot(snapshot *Snapshot) (err error) {
|
||||
if entry.Size != fileSize {
|
||||
err = fmt.Errorf("The file %s has a size of %d but the total size of chunks is %d",
|
||||
entry.Path, entry.Size, fileSize)
|
||||
return false
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
@@ -2594,12 +2633,29 @@ func (manager *SnapshotManager) DownloadFile(path string, derivationKey string)
|
||||
derivationKey = derivationKey[len(derivationKey)-64:]
|
||||
}
|
||||
|
||||
err = manager.fileChunk.Decrypt(manager.config.FileKey, derivationKey)
|
||||
err, rewriteNeeded := manager.fileChunk.Decrypt(manager.config.FileKey, derivationKey)
|
||||
if err != nil {
|
||||
LOG_ERROR("DOWNLOAD_DECRYPT", "Failed to decrypt the file %s: %v", path, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if rewriteNeeded && manager.chunkOperator.rewriteChunks {
|
||||
|
||||
newChunk := manager.config.GetChunk()
|
||||
newChunk.Reset(true)
|
||||
newChunk.Write(manager.fileChunk.GetBytes())
|
||||
err = newChunk.Encrypt(manager.config.FileKey, derivationKey, true)
|
||||
if err == nil {
|
||||
err = manager.storage.UploadFile(0, path, newChunk.GetBytes())
|
||||
if err != nil {
|
||||
LOG_WARN("DOWNLOAD_REWRITE", "Failed to re-uploaded the file %s: %v", path, err)
|
||||
} else {
|
||||
LOG_INFO("DOWNLOAD_REWRITE", "The file %s has been re-uploaded", path)
|
||||
}
|
||||
}
|
||||
manager.config.PutChunk(newChunk)
|
||||
}
|
||||
|
||||
err = manager.snapshotCache.UploadFile(0, path, manager.fileChunk.GetBytes())
|
||||
if err != nil {
|
||||
LOG_WARN("DOWNLOAD_FILE_CACHE", "Failed to add the file %s to the snapshot cache: %v", path, err)
|
||||
|
||||
@@ -116,7 +116,7 @@ func createTestSnapshotManager(testDir string) *SnapshotManager {
|
||||
|
||||
func uploadTestChunk(manager *SnapshotManager, content []byte) string {
|
||||
|
||||
chunkOperator := CreateChunkOperator(manager.config, manager.storage, nil, false, *testThreads, false)
|
||||
chunkOperator := CreateChunkOperator(manager.config, manager.storage, nil, false, false, *testThreads, false)
|
||||
chunkOperator.UploadCompletionFunc = func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
|
||||
LOG_INFO("UPLOAD_CHUNK", "Chunk %s size %d uploaded", chunk.GetID(), chunkSize)
|
||||
}
|
||||
@@ -179,7 +179,7 @@ func createTestSnapshot(manager *SnapshotManager, snapshotID string, revision in
|
||||
|
||||
func checkTestSnapshots(manager *SnapshotManager, expectedSnapshots int, expectedFossils int) {
|
||||
|
||||
manager.CreateChunkOperator(false, 1, false)
|
||||
manager.CreateChunkOperator(false, false, 1, false)
|
||||
defer func() {
|
||||
manager.chunkOperator.Stop()
|
||||
manager.chunkOperator = nil
|
||||
@@ -625,7 +625,7 @@ func TestPruneNewSnapshots(t *testing.T) {
|
||||
// Now chunkHash1 wil be resurrected
|
||||
snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
|
||||
checkTestSnapshots(snapshotManager, 4, 0)
|
||||
snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3}, "", false, false, false, false, false, false, 1, false)
|
||||
snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3}, "", false, false, false, false, false, false, false, 1, false)
|
||||
}
|
||||
|
||||
// A fossil collection left by an aborted prune should be ignored if any supposedly deleted snapshot exists
|
||||
@@ -674,7 +674,7 @@ func TestPruneGhostSnapshots(t *testing.T) {
|
||||
// Run the prune again but the fossil collection should be igored, since revision 1 still exists
|
||||
snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
|
||||
checkTestSnapshots(snapshotManager, 3, 2)
|
||||
snapshotManager.CheckSnapshots("vm1@host1", []int{1, 2, 3}, "", false, false, false, false, true /*searchFossils*/, false, 1, false)
|
||||
snapshotManager.CheckSnapshots("vm1@host1", []int{1, 2, 3}, "", false, false, false, false, true /*searchFossils*/, false, false, 1, false)
|
||||
|
||||
// Prune snapshot 1 again
|
||||
snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
|
||||
@@ -688,5 +688,5 @@ func TestPruneGhostSnapshots(t *testing.T) {
|
||||
// Run the prune again and this time the fossil collection will be processed and the fossils removed
|
||||
snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
|
||||
checkTestSnapshots(snapshotManager, 3, 0)
|
||||
snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3, 4}, "", false, false, false, false, false, false, 1, false)
|
||||
snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3, 4}, "", false, false, false, false, false, false, false, 1, false)
|
||||
}
|
||||
|
||||
@@ -261,7 +261,8 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
|
||||
return fileStorage
|
||||
}
|
||||
|
||||
urlRegex := regexp.MustCompile(`^([\w-]+)://([\w\-@\.]+@)?([^/]+)(/(.+))?`)
|
||||
// Added \! to matched[2] because OneDrive drive ids contain ! (e.g. "b!xxx")
|
||||
urlRegex := regexp.MustCompile(`^([\w-]+)://([\w\-@\.\!]+@)?([^/]+)(/(.+))?`)
|
||||
|
||||
matched := urlRegex.FindStringSubmatch(storageURL)
|
||||
|
||||
@@ -644,15 +645,40 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
|
||||
SavePassword(preference, "gcd_token", tokenFile)
|
||||
return gcdStorage
|
||||
} else if matched[1] == "one" || matched[1] == "odb" {
|
||||
// Handle writing directly to the root of the drive
|
||||
// For odb://drive_id@/, drive_id@ is match[3] not match[2]
|
||||
if matched[2] == "" && strings.HasSuffix(matched[3], "@") {
|
||||
matched[2], matched[3] = matched[3], matched[2]
|
||||
}
|
||||
drive_id := matched[2]
|
||||
if len(drive_id) > 0 {
|
||||
drive_id = drive_id[:len(drive_id)-1]
|
||||
}
|
||||
storagePath := matched[3] + matched[4]
|
||||
prompt := fmt.Sprintf("Enter the path of the OneDrive token file (downloadable from https://duplicacy.com/one_start):")
|
||||
tokenFile := GetPassword(preference, matched[1] + "_token", prompt, true, resetPassword)
|
||||
oneDriveStorage, err := CreateOneDriveStorage(tokenFile, matched[1] == "odb", storagePath, threads)
|
||||
|
||||
// client_id, just like tokenFile, can be stored in preferences
|
||||
//prompt = fmt.Sprintf("Enter client_id for custom Azure app (if empty will use duplicacy.com one):")
|
||||
client_id := GetPasswordFromPreference(preference, matched[1] + "_client_id")
|
||||
client_secret := ""
|
||||
|
||||
if client_id != "" {
|
||||
// client_secret should go into keyring
|
||||
prompt = fmt.Sprintf("Enter client_secret for custom Azure app (if empty will use duplicacy.com one):")
|
||||
client_secret = GetPassword(preference, matched[1] + "_client_secret", prompt, true, resetPassword)
|
||||
}
|
||||
|
||||
oneDriveStorage, err := CreateOneDriveStorage(tokenFile, matched[1] == "odb", storagePath, threads, client_id, client_secret, drive_id)
|
||||
if err != nil {
|
||||
LOG_ERROR("STORAGE_CREATE", "Failed to load the OneDrive storage at %s: %v", storageURL, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
SavePassword(preference, matched[1] + "_token", tokenFile)
|
||||
if client_id != "" {
|
||||
SavePassword(preference, matched[1] + "_client_secret", client_secret)
|
||||
}
|
||||
return oneDriveStorage
|
||||
} else if matched[1] == "hubic" {
|
||||
storagePath := matched[3] + matched[4]
|
||||
@@ -730,7 +756,46 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
|
||||
LOG_ERROR("STORAGE_CREATE", "Failed to load the Storj storage at %s: %v", storageURL, err)
|
||||
return nil
|
||||
}
|
||||
SavePassword(preference, "storj_key", apiKey)
|
||||
SavePassword(preference, "storj_passphrase", passphrase)
|
||||
return storjStorage
|
||||
} else if matched[1] == "smb" {
|
||||
server := matched[3]
|
||||
username := matched[2]
|
||||
if username == "" {
|
||||
LOG_ERROR("STORAGE_CREATE", "No username is provided to access the SAMBA storage")
|
||||
return nil
|
||||
}
|
||||
username = username[:len(username)-1]
|
||||
storageDir := matched[5]
|
||||
port := 445
|
||||
|
||||
if strings.Contains(server, ":") {
|
||||
index := strings.Index(server, ":")
|
||||
port, _ = strconv.Atoi(server[index+1:])
|
||||
server = server[:index]
|
||||
}
|
||||
|
||||
if !strings.Contains(storageDir, "/") {
|
||||
LOG_ERROR("STORAGE_CREATE", "No share name specified for the SAMBA storage")
|
||||
return nil
|
||||
}
|
||||
|
||||
index := strings.Index(storageDir, "/")
|
||||
shareName := storageDir[:index]
|
||||
storageDir = storageDir[index+1:]
|
||||
|
||||
prompt := fmt.Sprintf("Enter the SAMBA password:")
|
||||
password := GetPassword(preference, "smb_password", prompt, true, resetPassword)
|
||||
sambaStorage, err := CreateSambaStorage(server, port, username, password, shareName, storageDir, threads)
|
||||
if err != nil {
|
||||
LOG_ERROR("STORAGE_CREATE", "Failed to load the SAMBA storage at %s: %v", storageURL, err)
|
||||
return nil
|
||||
}
|
||||
SavePassword(preference, "smb_password", password)
|
||||
return sambaStorage
|
||||
|
||||
|
||||
} else {
|
||||
LOG_ERROR("STORAGE_CREATE", "The storage type '%s' is not supported", matched[1])
|
||||
return nil
|
||||
|
||||
@@ -136,15 +136,15 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
|
||||
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
|
||||
return storage, err
|
||||
} else if *testStorageName == "one" {
|
||||
storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads)
|
||||
storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads, "", "", "")
|
||||
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
|
||||
return storage, err
|
||||
} else if *testStorageName == "odb" {
|
||||
storage, err := CreateOneDriveStorage(config["token_file"], true, config["storage_path"], threads)
|
||||
storage, err := CreateOneDriveStorage(config["token_file"], true, config["storage_path"], threads, "", "", "")
|
||||
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
|
||||
return storage, err
|
||||
} else if *testStorageName == "one" {
|
||||
storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads)
|
||||
storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads, "", "", "")
|
||||
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
|
||||
return storage, err
|
||||
} else if *testStorageName == "hubic" {
|
||||
@@ -176,6 +176,21 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
|
||||
}
|
||||
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
|
||||
return storage, err
|
||||
} else if *testStorageName == "storj" {
|
||||
storage, err := CreateStorjStorage(config["satellite"], config["key"], config["passphrase"], config["bucket"], config["storage_path"], threads)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
|
||||
return storage, err
|
||||
} else if *testStorageName == "smb" {
|
||||
port, _ := strconv.Atoi(config["port"])
|
||||
storage, err := CreateSambaStorage(config["server"], port, config["username"], config["password"], config["share"], config["storage_path"], threads)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
|
||||
return storage, err
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("Invalid storage named: %s", *testStorageName)
|
||||
|
||||
@@ -7,14 +7,15 @@ package duplicacy
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"runtime"
|
||||
|
||||
"github.com/gilbertchen/gopass"
|
||||
"golang.org/x/crypto/pbkdf2"
|
||||
@@ -56,7 +57,7 @@ func IsEmptyFilter(pattern string) bool {
|
||||
}
|
||||
|
||||
func IsUnspecifiedFilter(pattern string) bool {
|
||||
if pattern[0] != '+' && pattern[0] != '-' && !strings.HasPrefix(pattern, "i:") && !strings.HasPrefix(pattern, "e:") {
|
||||
if pattern[0] != '+' && pattern[0] != '-' && !strings.HasPrefix(pattern, "i:") && !strings.HasPrefix(pattern, "e:") {
|
||||
return true
|
||||
} else {
|
||||
return false
|
||||
@@ -275,7 +276,6 @@ func SavePassword(preference Preference, passwordType string, password string) {
|
||||
// The following code was modified from the online article 'Matching Wildcards: An Algorithm', by Kirk J. Krauss,
|
||||
// Dr. Dobb's, August 26, 2008. However, the version in the article doesn't handle cases like matching 'abcccd'
|
||||
// against '*ccd', and the version here fixed that issue.
|
||||
//
|
||||
func matchPattern(text string, pattern string) bool {
|
||||
|
||||
textLength := len(text)
|
||||
@@ -469,8 +469,39 @@ func PrintMemoryUsage() {
|
||||
runtime.ReadMemStats(&m)
|
||||
|
||||
LOG_INFO("MEMORY_STATS", "Currently allocated: %s, total allocated: %s, system memory: %s, number of GCs: %d",
|
||||
PrettySize(int64(m.Alloc)), PrettySize(int64(m.TotalAlloc)), PrettySize(int64(m.Sys)), m.NumGC)
|
||||
PrettySize(int64(m.Alloc)), PrettySize(int64(m.TotalAlloc)), PrettySize(int64(m.Sys)), m.NumGC)
|
||||
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (entry *Entry) dump() map[string]interface{} {
|
||||
|
||||
object := make(map[string]interface{})
|
||||
|
||||
object["path"] = entry.Path
|
||||
object["size"] = entry.Size
|
||||
object["time"] = entry.Time
|
||||
object["mode"] = entry.Mode
|
||||
object["hash"] = entry.Hash
|
||||
object["link"] = entry.Link
|
||||
|
||||
object["content"] = fmt.Sprintf("%d:%d:%d:%d",
|
||||
entry.StartChunk, entry.StartOffset, entry.EndChunk, entry.EndOffset)
|
||||
|
||||
if entry.UID != -1 && entry.GID != -1 {
|
||||
object["uid"] = entry.UID
|
||||
object["gid"] = entry.GID
|
||||
}
|
||||
|
||||
if entry.Attributes != nil && len(*entry.Attributes) > 0 {
|
||||
object["attributes"] = entry.Attributes
|
||||
}
|
||||
|
||||
return object
|
||||
}
|
||||
|
||||
func (entry *Entry) dumpString() string {
|
||||
data, _ := json.Marshal(entry.dump())
|
||||
return string(data)
|
||||
}
|
||||
|
||||
@@ -5,10 +5,34 @@
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func excludedByAttribute(attirbutes map[string][]byte) bool {
|
||||
value, ok := attirbutes["com.apple.metadata:com_apple_backup_excludeItem"]
|
||||
return ok && strings.Contains(string(value), "com.apple.backupd")
|
||||
func excludedByAttribute(attributes map[string][]byte) bool {
|
||||
value, ok := attributes["com.apple.metadata:com_apple_backup_excludeItem"]
|
||||
excluded := ok && strings.Contains(string(value), "com.apple.backupd")
|
||||
if !excluded {
|
||||
flags, ok := attributes[darwinFileFlagsKey]
|
||||
excluded = ok && (binary.LittleEndian.Uint32(flags)&unix.UF_NODUMP) != 0
|
||||
}
|
||||
return excluded
|
||||
}
|
||||
|
||||
func (entry *Entry) RestoreSpecial(fullPath string) error {
|
||||
mode := entry.Mode & uint32(fileModeMask)
|
||||
|
||||
if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
|
||||
mode |= unix.S_IFIFO
|
||||
} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
|
||||
mode |= unix.S_IFCHR
|
||||
} else if entry.Mode&uint32(os.ModeDevice) != 0 {
|
||||
mode |= unix.S_IFBLK
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
return unix.Mknod(fullPath, mode, int(entry.GetRdev()))
|
||||
}
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
)
|
||||
|
||||
func excludedByAttribute(attirbutes map[string][]byte) bool {
|
||||
_, ok := attirbutes["duplicacy_exclude"]
|
||||
return ok
|
||||
}
|
||||
@@ -5,9 +5,34 @@
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func excludedByAttribute(attirbutes map[string][]byte) bool {
|
||||
_, ok := attirbutes["duplicacy_exclude"]
|
||||
return ok
|
||||
func excludedByAttribute(attributes map[string][]byte) bool {
|
||||
_, excluded := attributes["user.duplicacy_exclude"]
|
||||
if !excluded {
|
||||
flags, ok := attributes[linuxFileFlagsKey]
|
||||
excluded = ok && (binary.LittleEndian.Uint32(flags)&linux_FS_NODUMP_FL) != 0
|
||||
}
|
||||
return excluded
|
||||
}
|
||||
|
||||
func (entry *Entry) RestoreSpecial(fullPath string) error {
|
||||
mode := entry.Mode & uint32(fileModeMask)
|
||||
|
||||
if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
|
||||
mode |= unix.S_IFIFO
|
||||
} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
|
||||
mode |= unix.S_IFCHR
|
||||
} else if entry.Mode&uint32(os.ModeDevice) != 0 {
|
||||
mode |= unix.S_IFBLK
|
||||
} else if entry.Mode&uint32(os.ModeSocket) != 0 {
|
||||
mode |= unix.S_IFSOCK
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
return unix.Mknod(fullPath, mode, int(entry.GetRdev()))
|
||||
}
|
||||
|
||||
@@ -2,18 +2,18 @@
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
//go:build !windows
|
||||
// +build !windows
|
||||
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
|
||||
"github.com/pkg/xattr"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func Readlink(path string) (isRegular bool, s string, err error) {
|
||||
@@ -21,24 +21,19 @@ func Readlink(path string) (isRegular bool, s string, err error) {
|
||||
return false, s, err
|
||||
}
|
||||
|
||||
func GetOwner(entry *Entry, fileInfo *os.FileInfo) {
|
||||
stat, ok := (*fileInfo).Sys().(*syscall.Stat_t)
|
||||
if ok && stat != nil {
|
||||
entry.UID = int(stat.Uid)
|
||||
entry.GID = int(stat.Gid)
|
||||
} else {
|
||||
entry.UID = -1
|
||||
entry.GID = -1
|
||||
}
|
||||
func GetOwner(entry *Entry, fileInfo os.FileInfo) {
|
||||
stat := fileInfo.Sys().(*syscall.Stat_t)
|
||||
entry.UID = int(stat.Uid)
|
||||
entry.GID = int(stat.Gid)
|
||||
}
|
||||
|
||||
func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
|
||||
stat, ok := (*fileInfo).Sys().(*syscall.Stat_t)
|
||||
if ok && stat != nil && (int(stat.Uid) != entry.UID || int(stat.Gid) != entry.GID) {
|
||||
func SetOwner(fullPath string, entry *Entry, fileInfo os.FileInfo) bool {
|
||||
stat := fileInfo.Sys().(*syscall.Stat_t)
|
||||
if (int(stat.Uid) != entry.UID || int(stat.Gid) != entry.GID) {
|
||||
if entry.UID != -1 && entry.GID != -1 {
|
||||
err := os.Lchown(fullPath, entry.UID, entry.GID)
|
||||
if err != nil {
|
||||
LOG_ERROR("RESTORE_CHOWN", "Failed to change uid or gid: %v", err)
|
||||
LOG_ERROR("RESTORE_CHOWN", "Failed to change uid or gid on %s: %v", entry.Path, err)
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -47,43 +42,67 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (entry *Entry) ReadAttributes(top string) {
|
||||
|
||||
fullPath := filepath.Join(top, entry.Path)
|
||||
attributes, _ := xattr.List(fullPath)
|
||||
if len(attributes) > 0 {
|
||||
entry.Attributes = &map[string][]byte{}
|
||||
for _, name := range attributes {
|
||||
attribute, err := xattr.Get(fullPath, name)
|
||||
if err == nil {
|
||||
(*entry.Attributes)[name] = attribute
|
||||
}
|
||||
}
|
||||
}
|
||||
type listEntryLinkKey struct {
|
||||
dev uint64
|
||||
ino uint64
|
||||
}
|
||||
|
||||
func (entry *Entry) SetAttributesToFile(fullPath string) {
|
||||
names, _ := xattr.List(fullPath)
|
||||
func (entry *Entry) getHardLinkKey(f os.FileInfo) (key listEntryLinkKey, linked bool) {
|
||||
if entry.IsDir() {
|
||||
return
|
||||
}
|
||||
stat := f.Sys().(*syscall.Stat_t)
|
||||
if stat.Nlink <= 1 {
|
||||
return
|
||||
}
|
||||
key.dev = uint64(stat.Dev)
|
||||
key.ino = uint64(stat.Ino)
|
||||
linked = true
|
||||
return
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
func (entry *Entry) ReadSpecial(fullPath string, fileInfo os.FileInfo) error {
|
||||
if fileInfo.Mode()&(os.ModeDevice|os.ModeCharDevice) == 0 {
|
||||
return nil
|
||||
}
|
||||
rdev := uint64(fileInfo.Sys().(*syscall.Stat_t).Rdev)
|
||||
entry.Size = 0
|
||||
entry.StartChunk = int(rdev & 0xFFFFFFFF)
|
||||
entry.StartOffset = int(rdev >> 32)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) GetRdev() uint64 {
|
||||
return uint64(entry.StartChunk) | uint64(entry.StartOffset)<<32
|
||||
}
|
||||
|
||||
newAttribute, found := (*entry.Attributes)[name]
|
||||
if found {
|
||||
oldAttribute, _ := xattr.Get(fullPath, name)
|
||||
if !bytes.Equal(oldAttribute, newAttribute) {
|
||||
xattr.Set(fullPath, name, newAttribute)
|
||||
}
|
||||
delete(*entry.Attributes, name)
|
||||
} else {
|
||||
xattr.Remove(fullPath, name)
|
||||
}
|
||||
func (entry *Entry) IsSameSpecial(fileInfo os.FileInfo) bool {
|
||||
stat := fileInfo.Sys().(*syscall.Stat_t)
|
||||
return (uint32(fileInfo.Mode()) == entry.Mode) && (uint64(stat.Rdev) == entry.GetRdev())
|
||||
}
|
||||
|
||||
func (entry *Entry) FmtSpecial() string {
|
||||
var c string
|
||||
mode := entry.Mode & uint32(os.ModeType)
|
||||
|
||||
if mode&uint32(os.ModeNamedPipe) != 0 {
|
||||
c = "p"
|
||||
} else if mode&uint32(os.ModeCharDevice) != 0 {
|
||||
c = "c"
|
||||
} else if mode&uint32(os.ModeDevice) != 0 {
|
||||
c = "b"
|
||||
} else if mode&uint32(os.ModeSocket) != 0 {
|
||||
c = "s"
|
||||
} else {
|
||||
return ""
|
||||
}
|
||||
|
||||
for name, attribute := range *entry.Attributes {
|
||||
xattr.Set(fullPath, name, attribute)
|
||||
}
|
||||
rdev := entry.GetRdev()
|
||||
return fmt.Sprintf("%s (%d, %d)", c, unix.Major(rdev), unix.Minor(rdev))
|
||||
}
|
||||
|
||||
func MakeHardlink(source string, target string) error {
|
||||
return unix.Linkat(unix.AT_FDCWD, source, unix.AT_FDCWD, target, 0)
|
||||
}
|
||||
|
||||
func joinPath(components ...string) string {
|
||||
|
||||
@@ -56,7 +56,11 @@ const (
|
||||
|
||||
// Readlink returns the destination of the named symbolic link.
|
||||
func Readlink(path string) (isRegular bool, s string, err error) {
|
||||
fd, err := syscall.CreateFile(syscall.StringToUTF16Ptr(path), FILE_READ_ATTRIBUTES,
|
||||
pPath, err := syscall.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
fd, err := syscall.CreateFile(pPath, FILE_READ_ATTRIBUTES,
|
||||
syscall.FILE_SHARE_READ, nil, syscall.OPEN_EXISTING,
|
||||
syscall.FILE_FLAG_OPEN_REPARSE_POINT|syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
|
||||
if err != nil {
|
||||
@@ -101,20 +105,17 @@ func Readlink(path string) (isRegular bool, s string, err error) {
|
||||
return false, s, nil
|
||||
}
|
||||
|
||||
func GetOwner(entry *Entry, fileInfo *os.FileInfo) {
|
||||
func GetOwner(entry *Entry, fileInfo os.FileInfo) {
|
||||
entry.UID = -1
|
||||
entry.GID = -1
|
||||
}
|
||||
|
||||
func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
|
||||
func SetOwner(fullPath string, entry *Entry, fileInfo os.FileInfo) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (entry *Entry) ReadAttributes(top string) {
|
||||
}
|
||||
|
||||
func (entry *Entry) SetAttributesToFile(fullPath string) {
|
||||
|
||||
func MakeHardlink(source string, target string) error {
|
||||
return os.Link(source, target)
|
||||
}
|
||||
|
||||
func joinPath(components ...string) string {
|
||||
@@ -132,6 +133,28 @@ func SplitDir(fullPath string) (dir string, file string) {
|
||||
return fullPath[:i+1], fullPath[i+1:]
|
||||
}
|
||||
|
||||
func excludedByAttribute(attirbutes map[string][]byte) bool {
|
||||
func excludedByAttribute(attributes map[string][]byte) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
type listEntryLinkKey struct{}
|
||||
|
||||
func (entry *Entry) getHardLinkKey(f os.FileInfo) (key listEntryLinkKey, linked bool) {
|
||||
return
|
||||
}
|
||||
|
||||
func (entry *Entry) ReadSpecial(fullPath string, fileInfo os.FileInfo) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) IsSameSpecial(fileInfo os.FileInfo) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (entry *Entry) RestoreSpecial(fullPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) FmtSpecial() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
38
src/duplicacy_utils_xbsd.go
Normal file
38
src/duplicacy_utils_xbsd.go
Normal file
@@ -0,0 +1,38 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
//go:build freebsd
|
||||
// +build freebsd
|
||||
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func excludedByAttribute(attributes map[string][]byte) bool {
|
||||
_, excluded := attributes["duplicacy_exclude"]
|
||||
if !excluded {
|
||||
flags, ok := attributes[bsdFileFlagsKey]
|
||||
excluded = ok && (binary.LittleEndian.Uint32(flags)&bsd_UF_NODUMP) != 0
|
||||
}
|
||||
return excluded
|
||||
}
|
||||
|
||||
func (entry *Entry) RestoreSpecial(fullPath string) error {
|
||||
mode := entry.Mode & uint32(fileModeMask)
|
||||
|
||||
if entry.Mode&uint32(os.ModeNamedPipe) != 0 {
|
||||
mode |= syscall.S_IFIFO
|
||||
} else if entry.Mode&uint32(os.ModeCharDevice) != 0 {
|
||||
mode |= syscall.S_IFCHR
|
||||
} else if entry.Mode&uint32(os.ModeDevice) != 0 {
|
||||
mode |= syscall.S_IFBLK
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
return syscall.Mknod(fullPath, mode, entry.GetRdev())
|
||||
}
|
||||
35
src/duplicacy_xattr.go
Normal file
35
src/duplicacy_xattr.go
Normal file
@@ -0,0 +1,35 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
package duplicacy
|
||||
|
||||
import "os"
|
||||
|
||||
func (entry *Entry) ReadAttributes(fi os.FileInfo, fullPath string, normalize bool) error {
|
||||
return entry.readAttributes(fi, fullPath, normalize)
|
||||
}
|
||||
|
||||
func (entry *Entry) GetFileFlags(fileInfo os.FileInfo) bool {
|
||||
return entry.getFileFlags(fileInfo)
|
||||
}
|
||||
|
||||
func (entry *Entry) ReadFileFlags(fileInfo os.FileInfo, fullPath string) error {
|
||||
return entry.readFileFlags(fileInfo, fullPath)
|
||||
}
|
||||
|
||||
func (entry *Entry) RestoreEarlyDirFlags(fullPath string, mask uint32) error {
|
||||
return entry.restoreEarlyDirFlags(fullPath, mask)
|
||||
}
|
||||
|
||||
func (entry *Entry) RestoreEarlyFileFlags(f *os.File, mask uint32) error {
|
||||
return entry.restoreEarlyFileFlags(f, mask)
|
||||
}
|
||||
|
||||
func (entry *Entry) RestoreLateFileFlags(fullPath string, fileInfo os.FileInfo, mask uint32) error {
|
||||
return entry.restoreLateFileFlags(fullPath, fileInfo, mask)
|
||||
}
|
||||
|
||||
func (entry *Entry) SetAttributesToFile(fullPath string, normalize bool) error {
|
||||
return entry.setAttributesToFile(fullPath, normalize)
|
||||
}
|
||||
149
src/duplicacy_xattr_darwin.go
Normal file
149
src/duplicacy_xattr_darwin.go
Normal file
@@ -0,0 +1,149 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"math"
|
||||
"os"
|
||||
"syscall"
|
||||
|
||||
"github.com/pkg/xattr"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const (
|
||||
darwinFileFlagsKey = "\x00bf"
|
||||
)
|
||||
|
||||
var darwinIsSuperUser bool
|
||||
|
||||
func init() {
|
||||
darwinIsSuperUser = unix.Geteuid() == 0
|
||||
}
|
||||
|
||||
func (entry *Entry) readAttributes(fi os.FileInfo, fullPath string, normalize bool) error {
|
||||
if entry.IsSpecial() {
|
||||
return nil
|
||||
}
|
||||
|
||||
attributes, err := xattr.LList(fullPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(attributes) > 0 {
|
||||
entry.Attributes = &map[string][]byte{}
|
||||
}
|
||||
var allErrors error
|
||||
for _, name := range attributes {
|
||||
value, err := xattr.LGet(fullPath, name)
|
||||
if err != nil {
|
||||
allErrors = errors.Join(allErrors, err)
|
||||
} else {
|
||||
(*entry.Attributes)[name] = value
|
||||
}
|
||||
}
|
||||
|
||||
return allErrors
|
||||
}
|
||||
|
||||
func (entry *Entry) getFileFlags(fileInfo os.FileInfo) bool {
|
||||
stat := fileInfo.Sys().(*syscall.Stat_t)
|
||||
if stat.Flags != 0 {
|
||||
if entry.Attributes == nil {
|
||||
entry.Attributes = &map[string][]byte{}
|
||||
}
|
||||
v := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(v, stat.Flags)
|
||||
(*entry.Attributes)[darwinFileFlagsKey] = v
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (entry *Entry) readFileFlags(fileInfo os.FileInfo, fullPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) setAttributesToFile(fullPath string, normalize bool) error {
|
||||
if entry.Attributes == nil || len(*entry.Attributes) == 0 || entry.IsSpecial() {
|
||||
return nil
|
||||
}
|
||||
attributes := *entry.Attributes
|
||||
|
||||
if _, haveFlags := attributes[darwinFileFlagsKey]; haveFlags && len(attributes) <= 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
names, err := xattr.LList(fullPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, name := range names {
|
||||
newAttribute, found := attributes[name]
|
||||
if found {
|
||||
oldAttribute, _ := xattr.LGet(fullPath, name)
|
||||
if !bytes.Equal(oldAttribute, newAttribute) {
|
||||
err = errors.Join(err, xattr.LSet(fullPath, name, newAttribute))
|
||||
}
|
||||
delete(attributes, name)
|
||||
} else {
|
||||
err = errors.Join(err, xattr.LRemove(fullPath, name))
|
||||
}
|
||||
}
|
||||
|
||||
for name, attribute := range attributes {
|
||||
if len(name) > 0 && name[0] == '\x00' {
|
||||
continue
|
||||
}
|
||||
err = errors.Join(err, xattr.LSet(fullPath, name, attribute))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (entry *Entry) restoreEarlyDirFlags(fullPath string, mask uint32) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) restoreEarlyFileFlags(f *os.File, mask uint32) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) restoreLateFileFlags(fullPath string, fileInfo os.FileInfo, mask uint32) error {
|
||||
if mask == math.MaxUint32 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if darwinIsSuperUser {
|
||||
mask |= ^uint32(unix.UF_SETTABLE | unix.SF_SETTABLE)
|
||||
} else {
|
||||
mask |= ^uint32(unix.UF_SETTABLE)
|
||||
}
|
||||
|
||||
var flags uint32
|
||||
|
||||
if entry.Attributes != nil {
|
||||
if v, have := (*entry.Attributes)[darwinFileFlagsKey]; have {
|
||||
flags = binary.LittleEndian.Uint32(v)
|
||||
}
|
||||
}
|
||||
|
||||
stat := fileInfo.Sys().(*syscall.Stat_t)
|
||||
|
||||
flags = (flags & ^mask) | (stat.Flags & mask)
|
||||
|
||||
if flags != stat.Flags {
|
||||
f, err := os.OpenFile(fullPath, os.O_RDONLY|unix.O_SYMLINK, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = unix.Fchflags(int(f.Fd()), int(flags))
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
234
src/duplicacy_xattr_linux.go
Normal file
234
src/duplicacy_xattr_linux.go
Normal file
@@ -0,0 +1,234 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"unsafe"
|
||||
|
||||
"github.com/pkg/xattr"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const (
|
||||
linux_FS_SECRM_FL = 0x00000001 /* Secure deletion */
|
||||
linux_FS_UNRM_FL = 0x00000002 /* Undelete */
|
||||
linux_FS_COMPR_FL = 0x00000004 /* Compress file */
|
||||
linux_FS_SYNC_FL = 0x00000008 /* Synchronous updates */
|
||||
linux_FS_IMMUTABLE_FL = 0x00000010 /* Immutable file */
|
||||
linux_FS_APPEND_FL = 0x00000020 /* writes to file may only append */
|
||||
linux_FS_NODUMP_FL = 0x00000040 /* do not dump file */
|
||||
linux_FS_NOATIME_FL = 0x00000080 /* do not update atime */
|
||||
linux_FS_NOCOMP_FL = 0x00000400 /* Don't compress */
|
||||
linux_FS_JOURNAL_DATA_FL = 0x00004000 /* Reserved for ext3 */
|
||||
linux_FS_NOTAIL_FL = 0x00008000 /* file tail should not be merged */
|
||||
linux_FS_DIRSYNC_FL = 0x00010000 /* dirsync behaviour (directories only) */
|
||||
linux_FS_TOPDIR_FL = 0x00020000 /* Top of directory hierarchies*/
|
||||
linux_FS_NOCOW_FL = 0x00800000 /* Do not cow file */
|
||||
linux_FS_PROJINHERIT_FL = 0x20000000 /* Create with parents projid */
|
||||
|
||||
linuxIocFlagsFileEarly = linux_FS_SECRM_FL | linux_FS_UNRM_FL | linux_FS_COMPR_FL | linux_FS_NODUMP_FL | linux_FS_NOATIME_FL | linux_FS_NOCOMP_FL | linux_FS_JOURNAL_DATA_FL | linux_FS_NOTAIL_FL | linux_FS_NOCOW_FL
|
||||
linuxIocFlagsDirEarly = linux_FS_TOPDIR_FL | linux_FS_PROJINHERIT_FL
|
||||
linuxIocFlagsLate = linux_FS_SYNC_FL | linux_FS_IMMUTABLE_FL | linux_FS_APPEND_FL | linux_FS_DIRSYNC_FL
|
||||
|
||||
linuxFileFlagsKey = "\x00lf"
|
||||
)
|
||||
|
||||
var (
|
||||
errENOTTY error = unix.ENOTTY
|
||||
)
|
||||
|
||||
func ignoringEINTR(fn func() error) (err error) {
|
||||
for {
|
||||
err = fn()
|
||||
if err != unix.EINTR {
|
||||
break
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func ioctl(f *os.File, request uintptr, attrp *uint32) error {
|
||||
return ignoringEINTR(func() error {
|
||||
argp := uintptr(unsafe.Pointer(attrp))
|
||||
|
||||
_, _, errno := unix.Syscall(unix.SYS_IOCTL, f.Fd(), request, argp)
|
||||
if errno == 0 {
|
||||
return nil
|
||||
} else if errno == unix.ENOTTY {
|
||||
return errENOTTY
|
||||
}
|
||||
return errno
|
||||
})
|
||||
}
|
||||
|
||||
func (entry *Entry) readAttributes(fi os.FileInfo, fullPath string, normalize bool) error {
|
||||
attributes, err := xattr.LList(fullPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(attributes) > 0 {
|
||||
entry.Attributes = &map[string][]byte{}
|
||||
}
|
||||
var allErrors error
|
||||
for _, name := range attributes {
|
||||
value, err := xattr.LGet(fullPath, name)
|
||||
if err != nil {
|
||||
allErrors = errors.Join(allErrors, err)
|
||||
} else {
|
||||
(*entry.Attributes)[name] = value
|
||||
}
|
||||
}
|
||||
|
||||
return allErrors
|
||||
}
|
||||
|
||||
func (entry *Entry) getFileFlags(fileInfo os.FileInfo) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (entry *Entry) readFileFlags(fileInfo os.FileInfo, fullPath string) error {
|
||||
// the linux file flags interface is quite depressing. The half assed attempt at statx
|
||||
// doesn't even cover the flags we're usually interested in for btrfs
|
||||
if !(entry.IsFile() || entry.IsDir()) {
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(fullPath, os.O_RDONLY|unix.O_NONBLOCK|unix.O_NOFOLLOW|unix.O_NOATIME, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var flags uint32
|
||||
|
||||
err = ioctl(f, unix.FS_IOC_GETFLAGS, &flags)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
// inappropriate ioctl for device means flags aren't a thing on that FS
|
||||
if err == unix.ENOTTY {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if flags != 0 {
|
||||
if entry.Attributes == nil {
|
||||
entry.Attributes = &map[string][]byte{}
|
||||
}
|
||||
v := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(v, flags)
|
||||
(*entry.Attributes)[linuxFileFlagsKey] = v
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) setAttributesToFile(fullPath string, normalize bool) error {
|
||||
if entry.Attributes == nil || len(*entry.Attributes) == 0 {
|
||||
return nil
|
||||
}
|
||||
attributes := *entry.Attributes
|
||||
|
||||
if _, haveFlags := attributes[linuxFileFlagsKey]; haveFlags && len(attributes) <= 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
names, err := xattr.LList(fullPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, name := range names {
|
||||
newAttribute, found := (*entry.Attributes)[name]
|
||||
if found {
|
||||
oldAttribute, _ := xattr.LGet(fullPath, name)
|
||||
if !bytes.Equal(oldAttribute, newAttribute) {
|
||||
err = errors.Join(err, xattr.LSet(fullPath, name, newAttribute))
|
||||
}
|
||||
delete(*entry.Attributes, name)
|
||||
} else {
|
||||
err = errors.Join(err, xattr.LRemove(fullPath, name))
|
||||
}
|
||||
}
|
||||
|
||||
for name, attribute := range *entry.Attributes {
|
||||
if len(name) > 0 && name[0] == '\x00' {
|
||||
continue
|
||||
}
|
||||
err = errors.Join(err, xattr.LSet(fullPath, name, attribute))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (entry *Entry) restoreEarlyDirFlags(fullPath string, mask uint32) error {
|
||||
if entry.Attributes == nil || mask == math.MaxUint32 {
|
||||
return nil
|
||||
}
|
||||
var flags uint32
|
||||
|
||||
if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
|
||||
flags = binary.LittleEndian.Uint32(v) & linuxIocFlagsDirEarly & ^mask
|
||||
}
|
||||
|
||||
if flags != 0 {
|
||||
f, err := os.OpenFile(fullPath, os.O_RDONLY|unix.O_DIRECTORY, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = ioctl(f, unix.FS_IOC_SETFLAGS, &flags)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Set flags 0x%.8x failed: %w", flags, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) restoreEarlyFileFlags(f *os.File, mask uint32) error {
|
||||
if entry.Attributes == nil || mask == math.MaxUint32 {
|
||||
return nil
|
||||
}
|
||||
var flags uint32
|
||||
|
||||
if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
|
||||
flags = binary.LittleEndian.Uint32(v) & linuxIocFlagsFileEarly & ^mask
|
||||
}
|
||||
|
||||
if flags != 0 {
|
||||
err := ioctl(f, unix.FS_IOC_SETFLAGS, &flags)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Set flags 0x%.8x failed: %w", flags, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) restoreLateFileFlags(fullPath string, fileInfo os.FileInfo, mask uint32) error {
|
||||
if entry.IsLink() || entry.Attributes == nil || mask == math.MaxUint32 {
|
||||
return nil
|
||||
}
|
||||
var flags uint32
|
||||
|
||||
if v, have := (*entry.Attributes)[linuxFileFlagsKey]; have {
|
||||
flags = binary.LittleEndian.Uint32(v) & (linuxIocFlagsFileEarly | linuxIocFlagsDirEarly | linuxIocFlagsLate) & ^mask
|
||||
}
|
||||
|
||||
if flags != 0 {
|
||||
f, err := os.OpenFile(fullPath, os.O_RDONLY|unix.O_NOFOLLOW, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = ioctl(f, unix.FS_IOC_SETFLAGS, &flags)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Set flags 0x%.8x failed: %w", flags, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
35
src/duplicacy_xattr_windows.go
Normal file
35
src/duplicacy_xattr_windows.go
Normal file
@@ -0,0 +1,35 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
package duplicacy
|
||||
|
||||
import "os"
|
||||
|
||||
func (entry *Entry) readAttributes(fi os.FileInfo, fullPath string, normalize bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) getFileFlags(fileInfo os.FileInfo) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (entry *Entry) readFileFlags(fileInfo os.FileInfo, fullPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) setAttributesToFile(fullPath string, normalize bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) restoreEarlyDirFlags(fullPath string, mask uint32) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) restoreEarlyFileFlags(f *os.File, mask uint32) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) restoreLateFileFlags(fullPath string, fileInfo os.FileInfo, mask uint32) error {
|
||||
return nil
|
||||
}
|
||||
155
src/duplicacy_xattr_xbsd.go
Normal file
155
src/duplicacy_xattr_xbsd.go
Normal file
@@ -0,0 +1,155 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Free for personal use and commercial trial
|
||||
// Commercial use requires per-user licenses available from https://duplicacy.com
|
||||
|
||||
//go:build freebsd || netbsd
|
||||
// +build freebsd netbsd
|
||||
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"math"
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
|
||||
"github.com/pkg/xattr"
|
||||
)
|
||||
|
||||
const (
|
||||
bsd_UF_NODUMP = 0x1
|
||||
bsd_SF_SETTABLE = 0xffff0000
|
||||
bsd_UF_SETTABLE = 0x0000ffff
|
||||
|
||||
bsdFileFlagsKey = "\x00bf"
|
||||
)
|
||||
|
||||
var bsdIsSuperUser bool
|
||||
|
||||
func init() {
|
||||
bsdIsSuperUser = syscall.Geteuid() == 0
|
||||
}
|
||||
|
||||
func (entry *Entry) readAttributes(fi os.FileInfo, fullPath string, normalize bool) error {
|
||||
if entry.IsSpecial() {
|
||||
return nil
|
||||
}
|
||||
|
||||
attributes, err := xattr.LList(fullPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(attributes) > 0 {
|
||||
entry.Attributes = &map[string][]byte{}
|
||||
}
|
||||
var allErrors error
|
||||
for _, name := range attributes {
|
||||
value, err := xattr.LGet(fullPath, name)
|
||||
if err != nil {
|
||||
allErrors = errors.Join(allErrors, err)
|
||||
} else {
|
||||
(*entry.Attributes)[name] = value
|
||||
}
|
||||
}
|
||||
|
||||
return allErrors
|
||||
}
|
||||
|
||||
func (entry *Entry) getFileFlags(fileInfo os.FileInfo) bool {
|
||||
stat := fileInfo.Sys().(*syscall.Stat_t)
|
||||
if stat.Flags != 0 {
|
||||
if entry.Attributes == nil {
|
||||
entry.Attributes = &map[string][]byte{}
|
||||
}
|
||||
v := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(v, stat.Flags)
|
||||
(*entry.Attributes)[bsdFileFlagsKey] = v
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (entry *Entry) readFileFlags(fileInfo os.FileInfo, fullPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) setAttributesToFile(fullPath string, normalize bool) error {
|
||||
if entry.Attributes == nil || len(*entry.Attributes) == 0 || entry.IsSpecial() {
|
||||
return nil
|
||||
}
|
||||
attributes := *entry.Attributes
|
||||
|
||||
if _, haveFlags := attributes[bsdFileFlagsKey]; haveFlags && len(attributes) <= 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
names, err := xattr.LList(fullPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, name := range names {
|
||||
newAttribute, found := attributes[name]
|
||||
if found {
|
||||
oldAttribute, _ := xattr.LGet(fullPath, name)
|
||||
if !bytes.Equal(oldAttribute, newAttribute) {
|
||||
err = errors.Join(err, xattr.LSet(fullPath, name, newAttribute))
|
||||
}
|
||||
delete(attributes, name)
|
||||
} else {
|
||||
err = errors.Join(err, xattr.LRemove(fullPath, name))
|
||||
}
|
||||
}
|
||||
|
||||
for name, attribute := range attributes {
|
||||
if len(name) > 0 && name[0] == '\x00' {
|
||||
continue
|
||||
}
|
||||
err = errors.Join(err, xattr.LSet(fullPath, name, attribute))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (entry *Entry) restoreEarlyDirFlags(fullPath string, mask uint32) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) restoreEarlyFileFlags(f *os.File, mask uint32) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (entry *Entry) restoreLateFileFlags(fullPath string, fileInfo os.FileInfo, mask uint32) error {
|
||||
if mask == math.MaxUint32 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if bsdIsSuperUser {
|
||||
mask |= ^uint32(bsd_UF_SETTABLE | bsd_SF_SETTABLE)
|
||||
} else {
|
||||
mask |= ^uint32(bsd_UF_SETTABLE)
|
||||
}
|
||||
|
||||
var flags uint32
|
||||
|
||||
if entry.Attributes != nil {
|
||||
if v, have := (*entry.Attributes)[bsdFileFlagsKey]; have {
|
||||
flags = binary.LittleEndian.Uint32(v)
|
||||
}
|
||||
}
|
||||
|
||||
stat := fileInfo.Sys().(*syscall.Stat_t)
|
||||
|
||||
flags = (flags & ^mask) | (stat.Flags & mask)
|
||||
|
||||
if flags != stat.Flags {
|
||||
pPath, _ := syscall.BytePtrFromString(fullPath)
|
||||
if _, _, errno := syscall.Syscall(syscall.SYS_LCHFLAGS,
|
||||
uintptr(unsafe.Pointer(pPath)),
|
||||
uintptr(flags), 0); errno != 0 {
|
||||
return os.NewSyscallError("lchflags", errno)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user