|
|
|
@@ -187,7 +187,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
|
|
|
|
|
|
|
|
|
|
|
// If the listing operation is fast and this is an initial backup, list all chunks and
|
|
|
|
// If the listing operation is fast and this is an initial backup, list all chunks and
|
|
|
|
// put them in the cache.
|
|
|
|
// put them in the cache.
|
|
|
|
if (manager.storage.IsFastListing() && remoteSnapshot.Revision == 0) {
|
|
|
|
if manager.storage.IsFastListing() && remoteSnapshot.Revision == 0 {
|
|
|
|
LOG_INFO("BACKUP_LIST", "Listing all chunks")
|
|
|
|
LOG_INFO("BACKUP_LIST", "Listing all chunks")
|
|
|
|
allChunks, _ := manager.SnapshotManager.ListAllFiles(manager.storage, "chunks/")
|
|
|
|
allChunks, _ := manager.SnapshotManager.ListAllFiles(manager.storage, "chunks/")
|
|
|
|
|
|
|
|
|
|
|
|
@@ -239,7 +239,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
|
|
|
// List local files
|
|
|
|
// List local files
|
|
|
|
defer CatchLogException()
|
|
|
|
defer CatchLogException()
|
|
|
|
localSnapshot.ListLocalFiles(shadowTop, manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute, localListingChannel, &skippedDirectories, &skippedFiles)
|
|
|
|
localSnapshot.ListLocalFiles(shadowTop, manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute, localListingChannel, &skippedDirectories, &skippedFiles)
|
|
|
|
} ()
|
|
|
|
}()
|
|
|
|
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
go func() {
|
|
|
|
// List remote files
|
|
|
|
// List remote files
|
|
|
|
@@ -261,7 +261,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
close(remoteListingChannel)
|
|
|
|
close(remoteListingChannel)
|
|
|
|
} ()
|
|
|
|
}()
|
|
|
|
|
|
|
|
|
|
|
|
// Create the local file list
|
|
|
|
// Create the local file list
|
|
|
|
localEntryList, err := CreateEntryList(manager.snapshotID, manager.cachePath, maximumInMemoryEntries)
|
|
|
|
localEntryList, err := CreateEntryList(manager.snapshotID, manager.cachePath, maximumInMemoryEntries)
|
|
|
|
@@ -275,7 +275,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
|
|
|
var remoteEntry *Entry
|
|
|
|
var remoteEntry *Entry
|
|
|
|
remoteListingOK := true
|
|
|
|
remoteListingOK := true
|
|
|
|
for {
|
|
|
|
for {
|
|
|
|
localEntry := <- localListingChannel
|
|
|
|
localEntry := <-localListingChannel
|
|
|
|
if localEntry == nil {
|
|
|
|
if localEntry == nil {
|
|
|
|
break
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@@ -289,7 +289,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
|
|
|
compareResult = localEntry.Compare(remoteEntry)
|
|
|
|
compareResult = localEntry.Compare(remoteEntry)
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
if remoteListingOK {
|
|
|
|
if remoteListingOK {
|
|
|
|
remoteEntry, remoteListingOK = <- remoteListingChannel
|
|
|
|
remoteEntry, remoteListingOK = <-remoteListingChannel
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if !remoteListingOK {
|
|
|
|
if !remoteListingOK {
|
|
|
|
compareResult = -1
|
|
|
|
compareResult = -1
|
|
|
|
@@ -448,7 +448,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
|
|
|
|
|
|
|
|
|
|
|
_, found := chunkCache[chunkID]
|
|
|
|
_, found := chunkCache[chunkID]
|
|
|
|
if found {
|
|
|
|
if found {
|
|
|
|
if time.Now().Unix() - lastUploadingTime > keepUploadAlive {
|
|
|
|
if time.Now().Unix()-lastUploadingTime > keepUploadAlive {
|
|
|
|
LOG_INFO("UPLOAD_KEEPALIVE", "Skip chunk cache to keep connection alive")
|
|
|
|
LOG_INFO("UPLOAD_KEEPALIVE", "Skip chunk cache to keep connection alive")
|
|
|
|
found = false
|
|
|
|
found = false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@@ -558,7 +558,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
|
|
|
|
if showStatistics {
|
|
|
|
if showStatistics {
|
|
|
|
|
|
|
|
|
|
|
|
LOG_INFO("BACKUP_STATS", "Files: %d total, %s bytes; %d new, %s bytes",
|
|
|
|
LOG_INFO("BACKUP_STATS", "Files: %d total, %s bytes; %d new, %s bytes",
|
|
|
|
localEntryList.NumberOfEntries - int64(len(skippedFiles)),
|
|
|
|
localEntryList.NumberOfEntries-int64(len(skippedFiles)),
|
|
|
|
PrettyNumber(preservedFileSize+uploadedFileSize),
|
|
|
|
PrettyNumber(preservedFileSize+uploadedFileSize),
|
|
|
|
len(localEntryList.ModifiedEntries), PrettyNumber(uploadedFileSize))
|
|
|
|
len(localEntryList.ModifiedEntries), PrettyNumber(uploadedFileSize))
|
|
|
|
|
|
|
|
|
|
|
|
@@ -686,7 +686,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
|
|
|
// List local files
|
|
|
|
// List local files
|
|
|
|
defer CatchLogException()
|
|
|
|
defer CatchLogException()
|
|
|
|
localSnapshot.ListLocalFiles(top, manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute, localListingChannel, nil, nil)
|
|
|
|
localSnapshot.ListLocalFiles(top, manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute, localListingChannel, nil, nil)
|
|
|
|
} ()
|
|
|
|
}()
|
|
|
|
|
|
|
|
|
|
|
|
remoteSnapshot := manager.SnapshotManager.DownloadSnapshot(manager.snapshotID, revision)
|
|
|
|
remoteSnapshot := manager.SnapshotManager.DownloadSnapshot(manager.snapshotID, revision)
|
|
|
|
manager.SnapshotManager.DownloadSnapshotSequences(remoteSnapshot)
|
|
|
|
manager.SnapshotManager.DownloadSnapshotSequences(remoteSnapshot)
|
|
|
|
@@ -698,7 +698,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
})
|
|
|
|
})
|
|
|
|
close(remoteListingChannel)
|
|
|
|
close(remoteListingChannel)
|
|
|
|
} ()
|
|
|
|
}()
|
|
|
|
|
|
|
|
|
|
|
|
var localEntry *Entry
|
|
|
|
var localEntry *Entry
|
|
|
|
localListingOK := true
|
|
|
|
localListingOK := true
|
|
|
|
@@ -748,7 +748,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
|
|
|
|
|
|
|
|
|
|
|
for {
|
|
|
|
for {
|
|
|
|
if localEntry == nil && localListingOK {
|
|
|
|
if localEntry == nil && localListingOK {
|
|
|
|
localEntry, localListingOK = <- localListingChannel
|
|
|
|
localEntry, localListingOK = <-localListingChannel
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if localEntry == nil {
|
|
|
|
if localEntry == nil {
|
|
|
|
compareResult = 1
|
|
|
|
compareResult = 1
|
|
|
|
@@ -882,7 +882,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
for localListingOK {
|
|
|
|
for localListingOK {
|
|
|
|
localEntry, localListingOK = <- localListingChannel
|
|
|
|
localEntry, localListingOK = <-localListingChannel
|
|
|
|
if localEntry != nil {
|
|
|
|
if localEntry != nil {
|
|
|
|
extraFiles = append(extraFiles, localEntry.Path)
|
|
|
|
extraFiles = append(extraFiles, localEntry.Path)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@@ -1298,6 +1298,7 @@ func (manager *BackupManager) UploadSnapshot(chunkOperator *ChunkOperator, top s
|
|
|
|
// file under the .duplicacy directory and then replaces the existing one. Otherwise, the existing file will be
|
|
|
|
// file under the .duplicacy directory and then replaces the existing one. Otherwise, the existing file will be
|
|
|
|
// overwritten directly.
|
|
|
|
// overwritten directly.
|
|
|
|
// Return: true, nil: Restored file;
|
|
|
|
// Return: true, nil: Restored file;
|
|
|
|
|
|
|
|
//
|
|
|
|
// false, nil: Skipped file;
|
|
|
|
// false, nil: Skipped file;
|
|
|
|
// false, error: Failure to restore file (only if allowFailures == true)
|
|
|
|
// false, error: Failure to restore file (only if allowFailures == true)
|
|
|
|
func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chunkMaker *ChunkMaker, entry *Entry, top string, inPlace bool, overwrite bool,
|
|
|
|
func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chunkMaker *ChunkMaker, entry *Entry, top string, inPlace bool, overwrite bool,
|
|
|
|
@@ -1876,7 +1877,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
LOG_INFO("SNAPSHOT_COPY", "Chunks to copy: %d, to skip: %d, total: %d", len(chunksToCopy), len(chunks) - len(chunksToCopy), len(chunks))
|
|
|
|
LOG_INFO("SNAPSHOT_COPY", "Chunks to copy: %d, to skip: %d, total: %d", len(chunksToCopy), len(chunks)-len(chunksToCopy), len(chunks))
|
|
|
|
|
|
|
|
|
|
|
|
chunkDownloader := CreateChunkOperator(manager.config, manager.storage, nil, false, false, downloadingThreads, false)
|
|
|
|
chunkDownloader := CreateChunkOperator(manager.config, manager.storage, nil, false, false, downloadingThreads, false)
|
|
|
|
|
|
|
|
|
|
|
|
@@ -1896,10 +1897,10 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
|
|
|
|
|
|
|
|
|
|
|
|
elapsedTime := time.Now().Sub(startTime).Seconds()
|
|
|
|
elapsedTime := time.Now().Sub(startTime).Seconds()
|
|
|
|
speed := int64(float64(atomic.LoadInt64(&uploadedBytes)) / elapsedTime)
|
|
|
|
speed := int64(float64(atomic.LoadInt64(&uploadedBytes)) / elapsedTime)
|
|
|
|
remainingTime := int64(float64(len(chunksToCopy) - chunkIndex - 1) / float64(chunkIndex + 1) * elapsedTime)
|
|
|
|
remainingTime := int64(float64(len(chunksToCopy)-chunkIndex-1) / float64(chunkIndex+1) * elapsedTime)
|
|
|
|
percentage := float64(chunkIndex + 1) / float64(len(chunksToCopy)) * 100.0
|
|
|
|
percentage := float64(chunkIndex+1) / float64(len(chunksToCopy)) * 100.0
|
|
|
|
LOG_INFO("COPY_PROGRESS", "%s chunk %s (%d/%d) %sB/s %s %.1f%%",
|
|
|
|
LOG_INFO("COPY_PROGRESS", "%s chunk %s (%d/%d) %sB/s %s %.1f%%",
|
|
|
|
action, chunk.GetID(), chunkIndex + 1, len(chunksToCopy),
|
|
|
|
action, chunk.GetID(), chunkIndex+1, len(chunksToCopy),
|
|
|
|
PrettySize(speed), PrettyTime(remainingTime), percentage)
|
|
|
|
PrettySize(speed), PrettyTime(remainingTime), percentage)
|
|
|
|
otherManager.config.PutChunk(chunk)
|
|
|
|
otherManager.config.PutChunk(chunk)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@@ -1923,7 +1924,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
|
|
|
|
chunkDownloader.Stop()
|
|
|
|
chunkDownloader.Stop()
|
|
|
|
chunkUploader.Stop()
|
|
|
|
chunkUploader.Stop()
|
|
|
|
|
|
|
|
|
|
|
|
LOG_INFO("SNAPSHOT_COPY", "Copied %d new chunks and skipped %d existing chunks", copiedChunks, len(chunks) - copiedChunks)
|
|
|
|
LOG_INFO("SNAPSHOT_COPY", "Copied %d new chunks and skipped %d existing chunks", copiedChunks, len(chunks)-copiedChunks)
|
|
|
|
|
|
|
|
|
|
|
|
for _, snapshot := range snapshots {
|
|
|
|
for _, snapshot := range snapshots {
|
|
|
|
if revisionMap[snapshot.ID][snapshot.Revision] == false {
|
|
|
|
if revisionMap[snapshot.ID][snapshot.Revision] == false {
|
|
|
|
|