From d3cea2c7d0a055945492191f2c6ed203d049bdfd Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Fri, 15 Sep 2017 23:34:06 -0400 Subject: [PATCH 1/5] print additional, table-formatted stats for CHECK --- src/duplicacy_snapshotmanager.go | 62 ++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index 4c984dc..749865a 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -8,12 +8,14 @@ import ( "io" "os" "fmt" + "text/tabwriter" "sort" "bytes" "regexp" "strconv" "strings" "time" + "math" "path" "io/ioutil" "encoding/json" @@ -896,10 +898,25 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe if showStatistics { - for snapshotID, snapshotList := range snapshotMap { + tableBuffer := new(bytes.Buffer) + tableWriter := tabwriter.NewWriter(tableBuffer, 0, 0, 1, ' ', tabwriter.AlignRight|tabwriter.Debug) + for snapshotID, snapshotList := range snapshotMap { + fmt.Fprintln(tableWriter, "") + fmt.Fprintln(tableWriter, " files \tbytes \tchunks \tbytes \tuniq \tbytes \tnew \tbytes \tsnap \trev \t \t") snapshotChunks := make(map[string]bool) + earliestSeenChunks := make(map[string]int) + + for _, snapshot := range snapshotList { + for _, chunkID := range manager.GetSnapshotChunks(snapshot) { + if earliestSeenChunks[chunkID] == 0 { + earliestSeenChunks[chunkID] = math.MaxInt64 + } + earliestSeenChunks[chunkID] = min(earliestSeenChunks[chunkID], snapshot.Revision) + } + } + for _, snapshot := range snapshotList { chunks := make(map[string]bool) @@ -910,42 +927,73 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe var totalChunkSize int64 var uniqueChunkSize int64 + var totalChunkCount int64 + var uniqueChunkCount int64 + var newChunkCount int64 + var newChunkSize int64 for chunkID, _ := range chunks { chunkSize := chunkSizeMap[chunkID] totalChunkSize += chunkSize + totalChunkCount += 1 + if earliestSeenChunks[chunkID] == snapshot.Revision { + newChunkCount += 1 + newChunkSize += chunkSize + } if chunkUniqueMap[chunkID] { uniqueChunkSize += chunkSize + uniqueChunkCount += 1 } } - files := "" + files := " \t " if snapshot.FileSize != 0 && snapshot.NumberOfFiles != 0 { - files = fmt.Sprintf("%d files (%s bytes), ", snapshot.NumberOfFiles, PrettyNumber(snapshot.FileSize)) + files = fmt.Sprintf("%d \t%s", snapshot.NumberOfFiles, PrettyNumber(snapshot.FileSize)) } - LOG_INFO("SNAPSHOT_CHECK", "Snapshot %s at revision %d: %s%s total chunk bytes, %s unique chunk bytes", - snapshot.ID, snapshot.Revision, files, PrettyNumber(totalChunkSize), PrettyNumber(uniqueChunkSize)) + creationTime := time.Unix(snapshot.StartTime, 0).Format("2006-01-02 15:04") + tagWithSpace := "" + if len(snapshot.Tag) > 0 { + tagWithSpace = snapshot.Tag + " " + } + fmt.Fprintln(tableWriter, fmt.Sprintf( + "%s \t%d \t%s \t%d \t%s \t%d \t%s \t%s \t%d \t@ %s %s%s \t", + files, totalChunkCount, PrettyNumber(totalChunkSize), uniqueChunkCount, PrettyNumber(uniqueChunkSize), newChunkCount, PrettyNumber(newChunkSize), snapshotID, snapshot.Revision, creationTime, tagWithSpace, snapshot.Options)) } var totalChunkSize int64 var uniqueChunkSize int64 + var totalChunkCount int64 + var uniqueChunkCount int64 for chunkID, _ := range snapshotChunks { chunkSize := chunkSizeMap[chunkID] totalChunkSize += chunkSize + totalChunkCount += 1 if chunkSnapshotMap[chunkID] != -1 { uniqueChunkSize += chunkSize + uniqueChunkCount += 1 } } - LOG_INFO("SNAPSHOT_CHECK", "Snapshot %s all revisions: %s total chunk bytes, %s unique chunk bytes", - snapshotID, PrettyNumber(totalChunkSize), PrettyNumber(uniqueChunkSize)) + fmt.Fprintln(tableWriter, fmt.Sprintf( + " \t \t%d \t%s \t%d \t%s \t \t \t%s \tall \t \t", + totalChunkCount, PrettyNumber(totalChunkSize), uniqueChunkCount, PrettyNumber(uniqueChunkSize), snapshotID)) } + tableWriter.Flush() + LOG_INFO("SNAPSHOT_CHECK", tableBuffer.String()) } return true } +func min(x, y int) int { + if x < y { + return x + } + return y +} + + // ConvertSequence converts a sequence of chunk hashes into a sequence of chunk ids. func (manager *SnapshotManager) ConvertSequence(sequence []string) (result []string) { result = make([]string, len(sequence)) From e2fe57e959f04d967231bed67914f9f6b6d2f805 Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Tue, 19 Sep 2017 00:59:38 -0400 Subject: [PATCH 2/5] reorder check -stats columns --- src/duplicacy_snapshotmanager.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index 749865a..5434faa 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -903,7 +903,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe for snapshotID, snapshotList := range snapshotMap { fmt.Fprintln(tableWriter, "") - fmt.Fprintln(tableWriter, " files \tbytes \tchunks \tbytes \tuniq \tbytes \tnew \tbytes \tsnap \trev \t \t") + fmt.Fprintln(tableWriter, " snap \trev \t \tfiles \tbytes \tchunks \tbytes \tuniq \tbytes \tnew \tbytes \t") snapshotChunks := make(map[string]bool) earliestSeenChunks := make(map[string]int) @@ -951,13 +951,9 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe files = fmt.Sprintf("%d \t%s", snapshot.NumberOfFiles, PrettyNumber(snapshot.FileSize)) } creationTime := time.Unix(snapshot.StartTime, 0).Format("2006-01-02 15:04") - tagWithSpace := "" - if len(snapshot.Tag) > 0 { - tagWithSpace = snapshot.Tag + " " - } fmt.Fprintln(tableWriter, fmt.Sprintf( - "%s \t%d \t%s \t%d \t%s \t%d \t%s \t%s \t%d \t@ %s %s%s \t", - files, totalChunkCount, PrettyNumber(totalChunkSize), uniqueChunkCount, PrettyNumber(uniqueChunkSize), newChunkCount, PrettyNumber(newChunkSize), snapshotID, snapshot.Revision, creationTime, tagWithSpace, snapshot.Options)) + "%s \t%d \t@ %s %5s \t%s \t%d \t%s \t%d \t%s \t%d \t%s \t", + snapshotID, snapshot.Revision, creationTime, snapshot.Options, files, totalChunkCount, PrettyNumber(totalChunkSize), uniqueChunkCount, PrettyNumber(uniqueChunkSize), newChunkCount, PrettyNumber(newChunkSize))) } var totalChunkSize int64 @@ -975,8 +971,8 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe } } fmt.Fprintln(tableWriter, fmt.Sprintf( - " \t \t%d \t%s \t%d \t%s \t \t \t%s \tall \t \t", - totalChunkCount, PrettyNumber(totalChunkSize), uniqueChunkCount, PrettyNumber(uniqueChunkSize), snapshotID)) + "%s \tall \t \t \t \t%d \t%s \t%d \t%s \t \t \t", + snapshotID, totalChunkCount, PrettyNumber(totalChunkSize), uniqueChunkCount, PrettyNumber(uniqueChunkSize))) } tableWriter.Flush() LOG_INFO("SNAPSHOT_CHECK", tableBuffer.String()) From f1fe64b9ccce27591f2d3027a001040a78272dc6 Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Tue, 19 Sep 2017 01:06:08 -0400 Subject: [PATCH 3/5] moving "func min" to MinInt in utils --- src/duplicacy_snapshotmanager.go | 10 +--------- src/duplicacy_utils.go | 7 +++++++ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index 5434faa..9f3137f 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -913,7 +913,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe if earliestSeenChunks[chunkID] == 0 { earliestSeenChunks[chunkID] = math.MaxInt64 } - earliestSeenChunks[chunkID] = min(earliestSeenChunks[chunkID], snapshot.Revision) + earliestSeenChunks[chunkID] = MinInt(earliestSeenChunks[chunkID], snapshot.Revision) } } @@ -982,14 +982,6 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe } -func min(x, y int) int { - if x < y { - return x - } - return y -} - - // ConvertSequence converts a sequence of chunk hashes into a sequence of chunk ids. func (manager *SnapshotManager) ConvertSequence(sequence []string) (result []string) { result = make([]string, len(sequence)) diff --git a/src/duplicacy_utils.go b/src/duplicacy_utils.go index 5c9187d..e3f5222 100644 --- a/src/duplicacy_utils.go +++ b/src/duplicacy_utils.go @@ -427,3 +427,10 @@ func AtoSize(sizeString string) (int) { return size } + +func MinInt(x, y int) (int) { + if x < y { + return x + } + return y +} From d5d76490416b5fb5449f6510703bcacdaf91a68d Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Tue, 19 Sep 2017 01:13:06 -0400 Subject: [PATCH 4/5] added -tabular to check options --- duplicacy/duplicacy_main.go | 7 ++++++- src/duplicacy_snapshotmanager.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index 955fd74..fbf425b 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -808,12 +808,13 @@ func checkSnapshots(context *cli.Context) { } showStatistics := context.Bool("stats") + showTabular := context.Bool("tabular") checkFiles := context.Bool("files") searchFossils := context.Bool("fossils") resurrect := context.Bool("resurrect") backupManager.SetupSnapshotCache(preference.Name) - backupManager.SnapshotManager.CheckSnapshots(id, revisions, tag, showStatistics, checkFiles, searchFossils, resurrect) + backupManager.SnapshotManager.CheckSnapshots(id, revisions, tag, showStatistics, showTabular, checkFiles, searchFossils, resurrect) runScript(context, preference.Name, "post") } @@ -1355,6 +1356,10 @@ func main() { Name: "stats", Usage: "show deduplication statistics (imply -all and all revisions)", }, + cli.BoolFlag { + Name: "tabular", + Usage: "show tabular usage and deduplication statistics (imply -stats, -all, and all revisions)", + }, cli.StringFlag { Name: "storage", Usage: "retrieve snapshots from the specified storage", diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index 9f3137f..420f049 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -748,7 +748,7 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList } // ListSnapshots shows the information about a snapshot. -func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToCheck []int, tag string, showStatistics bool, +func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToCheck []int, tag string, showStatistics bool, showTabular bool, checkFiles bool, searchFossils bool, resurrect bool) bool { LOG_DEBUG("LIST_PARAMETERS", "id: %s, revisions: %v, tag: %s, showStatistics: %t, checkFiles: %t, searchFossils: %t, resurrect: %t", From 45bc778898ed24d93f35b19c5008c09710f8a756 Mon Sep 17 00:00:00 2001 From: Arno Hautala Date: Tue, 19 Sep 2017 02:07:35 -0400 Subject: [PATCH 5/5] restored original stats output, enabled option to switch to tabular --- src/duplicacy_snapshotmanager.go | 169 ++++++++++++++++++++----------- 1 file changed, 112 insertions(+), 57 deletions(-) diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index 420f049..bc8b83f 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -896,90 +896,145 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe snapshotIDIndex += 1 } + if showTabular { + manager.ShowStatisticsTabular(snapshotMap, chunkSizeMap, chunkUniqueMap, chunkSnapshotMap) + } else if showStatistics { + manager.ShowStatistics(snapshotMap, chunkSizeMap, chunkUniqueMap, chunkSnapshotMap) + } - if showStatistics { - tableBuffer := new(bytes.Buffer) - tableWriter := tabwriter.NewWriter(tableBuffer, 0, 0, 1, ' ', tabwriter.AlignRight|tabwriter.Debug) + return true +} - for snapshotID, snapshotList := range snapshotMap { - fmt.Fprintln(tableWriter, "") - fmt.Fprintln(tableWriter, " snap \trev \t \tfiles \tbytes \tchunks \tbytes \tuniq \tbytes \tnew \tbytes \t") - snapshotChunks := make(map[string]bool) +// Print snapshot and revision statistics +func (manager *SnapshotManager) ShowStatistics(snapshotMap map[string] [] *Snapshot, chunkSizeMap map[string]int64, chunkUniqueMap map[string]bool, + chunkSnapshotMap map[string]int) { + for snapshotID, snapshotList := range snapshotMap { - earliestSeenChunks := make(map[string]int) + snapshotChunks := make(map[string]bool) - for _, snapshot := range snapshotList { - for _, chunkID := range manager.GetSnapshotChunks(snapshot) { - if earliestSeenChunks[chunkID] == 0 { - earliestSeenChunks[chunkID] = math.MaxInt64 - } - earliestSeenChunks[chunkID] = MinInt(earliestSeenChunks[chunkID], snapshot.Revision) + for _, snapshot := range snapshotList { + + chunks := make(map[string]bool) + for _, chunkID := range manager.GetSnapshotChunks(snapshot) { + chunks[chunkID] = true + snapshotChunks[chunkID] = true + } + + var totalChunkSize int64 + var uniqueChunkSize int64 + + for chunkID, _ := range chunks { + chunkSize := chunkSizeMap[chunkID] + totalChunkSize += chunkSize + if chunkUniqueMap[chunkID] { + uniqueChunkSize += chunkSize } } - for _, snapshot := range snapshotList { + files := "" + if snapshot.FileSize != 0 && snapshot.NumberOfFiles != 0 { + files = fmt.Sprintf("%d files (%s bytes), ", snapshot.NumberOfFiles, PrettyNumber(snapshot.FileSize)) + } + LOG_INFO("SNAPSHOT_CHECK", "Snapshot %s at revision %d: %s%s total chunk bytes, %s unique chunk bytes", + snapshot.ID, snapshot.Revision, files, PrettyNumber(totalChunkSize), PrettyNumber(uniqueChunkSize)) + } - chunks := make(map[string]bool) - for _, chunkID := range manager.GetSnapshotChunks(snapshot) { - chunks[chunkID] = true - snapshotChunks[chunkID] = true + var totalChunkSize int64 + var uniqueChunkSize int64 + for chunkID, _ := range snapshotChunks { + chunkSize := chunkSizeMap[chunkID] + totalChunkSize += chunkSize + + if chunkSnapshotMap[chunkID] != -1 { + uniqueChunkSize += chunkSize + } + } + LOG_INFO("SNAPSHOT_CHECK", "Snapshot %s all revisions: %s total chunk bytes, %s unique chunk bytes", + snapshotID, PrettyNumber(totalChunkSize), PrettyNumber(uniqueChunkSize)) + } +} + +// Print snapshot and revision statistics in tabular format +func (manager *SnapshotManager) ShowStatisticsTabular(snapshotMap map[string] [] *Snapshot, chunkSizeMap map[string]int64, chunkUniqueMap map[string]bool, + chunkSnapshotMap map[string]int) { + tableBuffer := new(bytes.Buffer) + tableWriter := tabwriter.NewWriter(tableBuffer, 0, 0, 1, ' ', tabwriter.AlignRight|tabwriter.Debug) + + for snapshotID, snapshotList := range snapshotMap { + fmt.Fprintln(tableWriter, "") + fmt.Fprintln(tableWriter, " snap \trev \t \tfiles \tbytes \tchunks \tbytes \tuniq \tbytes \tnew \tbytes \t") + snapshotChunks := make(map[string]bool) + + earliestSeenChunks := make(map[string]int) + + for _, snapshot := range snapshotList { + for _, chunkID := range manager.GetSnapshotChunks(snapshot) { + if earliestSeenChunks[chunkID] == 0 { + earliestSeenChunks[chunkID] = math.MaxInt64 } + earliestSeenChunks[chunkID] = MinInt(earliestSeenChunks[chunkID], snapshot.Revision) + } + } - var totalChunkSize int64 - var uniqueChunkSize int64 - var totalChunkCount int64 - var uniqueChunkCount int64 - var newChunkCount int64 - var newChunkSize int64 + for _, snapshot := range snapshotList { - for chunkID, _ := range chunks { - chunkSize := chunkSizeMap[chunkID] - totalChunkSize += chunkSize - totalChunkCount += 1 - if earliestSeenChunks[chunkID] == snapshot.Revision { - newChunkCount += 1 - newChunkSize += chunkSize - } - if chunkUniqueMap[chunkID] { - uniqueChunkSize += chunkSize - uniqueChunkCount += 1 - } - } - - files := " \t " - if snapshot.FileSize != 0 && snapshot.NumberOfFiles != 0 { - files = fmt.Sprintf("%d \t%s", snapshot.NumberOfFiles, PrettyNumber(snapshot.FileSize)) - } - creationTime := time.Unix(snapshot.StartTime, 0).Format("2006-01-02 15:04") - fmt.Fprintln(tableWriter, fmt.Sprintf( - "%s \t%d \t@ %s %5s \t%s \t%d \t%s \t%d \t%s \t%d \t%s \t", - snapshotID, snapshot.Revision, creationTime, snapshot.Options, files, totalChunkCount, PrettyNumber(totalChunkSize), uniqueChunkCount, PrettyNumber(uniqueChunkSize), newChunkCount, PrettyNumber(newChunkSize))) + chunks := make(map[string]bool) + for _, chunkID := range manager.GetSnapshotChunks(snapshot) { + chunks[chunkID] = true + snapshotChunks[chunkID] = true } var totalChunkSize int64 var uniqueChunkSize int64 var totalChunkCount int64 var uniqueChunkCount int64 - for chunkID, _ := range snapshotChunks { + var newChunkCount int64 + var newChunkSize int64 + + for chunkID, _ := range chunks { chunkSize := chunkSizeMap[chunkID] totalChunkSize += chunkSize totalChunkCount += 1 - - if chunkSnapshotMap[chunkID] != -1 { + if earliestSeenChunks[chunkID] == snapshot.Revision { + newChunkCount += 1 + newChunkSize += chunkSize + } + if chunkUniqueMap[chunkID] { uniqueChunkSize += chunkSize uniqueChunkCount += 1 } } + + files := " \t " + if snapshot.FileSize != 0 && snapshot.NumberOfFiles != 0 { + files = fmt.Sprintf("%d \t%s", snapshot.NumberOfFiles, PrettyNumber(snapshot.FileSize)) + } + creationTime := time.Unix(snapshot.StartTime, 0).Format("2006-01-02 15:04") fmt.Fprintln(tableWriter, fmt.Sprintf( - "%s \tall \t \t \t \t%d \t%s \t%d \t%s \t \t \t", - snapshotID, totalChunkCount, PrettyNumber(totalChunkSize), uniqueChunkCount, PrettyNumber(uniqueChunkSize))) + "%s \t%d \t@ %s %5s \t%s \t%d \t%s \t%d \t%s \t%d \t%s \t", + snapshotID, snapshot.Revision, creationTime, snapshot.Options, files, totalChunkCount, PrettyNumber(totalChunkSize), uniqueChunkCount, PrettyNumber(uniqueChunkSize), newChunkCount, PrettyNumber(newChunkSize))) } - tableWriter.Flush() - LOG_INFO("SNAPSHOT_CHECK", tableBuffer.String()) + + var totalChunkSize int64 + var uniqueChunkSize int64 + var totalChunkCount int64 + var uniqueChunkCount int64 + for chunkID, _ := range snapshotChunks { + chunkSize := chunkSizeMap[chunkID] + totalChunkSize += chunkSize + totalChunkCount += 1 + + if chunkSnapshotMap[chunkID] != -1 { + uniqueChunkSize += chunkSize + uniqueChunkCount += 1 + } + } + fmt.Fprintln(tableWriter, fmt.Sprintf( + "%s \tall \t \t \t \t%d \t%s \t%d \t%s \t \t \t", + snapshotID, totalChunkCount, PrettyNumber(totalChunkSize), uniqueChunkCount, PrettyNumber(uniqueChunkSize))) } - - return true - + tableWriter.Flush() + LOG_INFO("SNAPSHOT_CHECK", tableBuffer.String()) } // ConvertSequence converts a sequence of chunk hashes into a sequence of chunk ids.