diff --git a/src/duplicacy_benchmark.go b/src/duplicacy_benchmark.go new file mode 100644 index 0000000..22018d9 --- /dev/null +++ b/src/duplicacy_benchmark.go @@ -0,0 +1,235 @@ +// Copyright (c) Acrosync LLC. All rights reserved. +// Free for personal use and commercial trial +// Commercial use requires per-user licenses available from https://duplicacy.com + +package duplicacy + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "math/rand" + "os" + "path/filepath" + "time" +) + +func benchmarkSplit(reader *bytes.Reader, fileSize int64, chunkSize int, compression bool, encryption bool, annotation string) { + + config := CreateConfig() + config.CompressionLevel = DEFAULT_COMPRESSION_LEVEL + config.AverageChunkSize = chunkSize + config.MaximumChunkSize = chunkSize * 4 + config.MinimumChunkSize = chunkSize / 4 + config.ChunkSeed = []byte("duplicacy") + + config.HashKey = DEFAULT_KEY + config.IDKey = DEFAULT_KEY + + maker := CreateChunkMaker(config, false) + + startTime := float64(time.Now().UnixNano()) / 1e9 + numberOfChunks := 0 + reader.Seek(0, os.SEEK_SET) + maker.ForEachChunk(reader, + func(chunk *Chunk, final bool) { + if compression { + key := "" + if encryption { + key = "0123456789abcdef0123456789abcdef" + } + err := chunk.Encrypt([]byte(key), "") + if err != nil { + LOG_ERROR("BENCHMARK_ENCRYPT", "Failed to encrypt the chunk: %v", err) + } + } + config.PutChunk(chunk) + numberOfChunks++ + }, + func(size int64, hash string) (io.Reader, bool) { + return nil, false + }) + + runningTime := float64(time.Now().UnixNano())/1e9 - startTime + speed := int64(float64(fileSize) / runningTime) + LOG_INFO("BENCHMARK_SPLIT", "Split %s bytes into %d chunks %s in %.2fs: %s/s", PrettySize(fileSize), numberOfChunks, annotation, + runningTime, PrettySize(speed)) + + return +} + +func benchmarkRun(threads int, chunkCount int, job func(threadIndex int, chunkIndex int)) { + indexChannel := make(chan int, chunkCount) + stopChannel := make(chan int, threads) + finishChannel := make(chan int, threads) + + // Start the uploading goroutines + for i := 0; i < threads; i++ { + go func(threadIndex int) { + defer CatchLogException() + for { + select { + case chunkIndex := <-indexChannel: + job(threadIndex, chunkIndex) + finishChannel <- 0 + case <-stopChannel: + return + } + } + }(i) + } + + for i := 0; i < chunkCount; i++ { + indexChannel <- i + } + + for i := 0; i < chunkCount; i++ { + <-finishChannel + } + + for i := 0; i < threads; i++ { + stopChannel <- 0 + } +} + +func Benchmark(localDirectory string, storage Storage, fileSize int64, chunkSize int, chunkCount int, uploadThreads int, downloadThreads int) bool { + + filename := filepath.Join(localDirectory, "benchmark.dat") + + defer func() { + os.Remove(filename) + }() + + LOG_INFO("BENCHMARK_GENERATE", "Generating %s byte random data in memory", PrettySize(fileSize)) + data := make([]byte, fileSize) + _, err := rand.Read(data) + if err != nil { + LOG_ERROR("BENCHMARK_RAND", "Failed to generate random data: %v", err) + return false + } + + startTime := float64(time.Now().UnixNano()) / 1e9 + LOG_INFO("BENCHMARK_WRITE", "Writing random data to local disk") + err = ioutil.WriteFile(filename, data, 0600) + if err != nil { + LOG_ERROR("BENCHMARK_WRITE", "Failed to write the random data: %v", err) + return false + } + + runningTime := float64(time.Now().UnixNano())/1e9 - startTime + speed := int64(float64(fileSize) / runningTime) + LOG_INFO("BENCHMARK_WRITE", "Wrote %s bytes in %.2fs: %s/s", PrettySize(fileSize), runningTime, PrettySize(speed)) + + startTime = float64(time.Now().UnixNano()) / 1e9 + LOG_INFO("BENCHMARK_READ", "Reading the random data from local disk") + file, err := os.Open(filename) + if err != nil { + LOG_ERROR("BENCHMARK_OPEN", "Failed to open the random data file: %v", err) + return false + } + segment := make([]byte, 1024*1024) + for err == nil { + _, err = file.Read(segment) + } + if err != io.EOF { + LOG_ERROR("BENCHMARK_OPEN", "Failed to read the random data file: %v", err) + return false + } + file.Close() + runningTime = float64(time.Now().UnixNano())/1e9 - startTime + speed = int64(float64(fileSize) / runningTime) + LOG_INFO("BENCHMARK_READ", "Read %s bytes in %.2fs: %s/s", PrettySize(fileSize), runningTime, PrettySize(speed)) + + buffer := bytes.NewReader(data) + benchmarkSplit(buffer, fileSize, chunkSize, false, false, "without compression/encryption") + benchmarkSplit(buffer, fileSize, chunkSize, true, false, "with compression but without encryption") + benchmarkSplit(buffer, fileSize, chunkSize, true, true, "with compression and encryption") + + storage.CreateDirectory(0, "benchmark") + existingFiles, _, err := storage.ListFiles(0, "benchmark/") + if err != nil { + LOG_ERROR("BENCHMARK_LIST", "Failed to list the benchmark directory: %v", err) + return false + } + + var existingChunks []string + for _, f := range existingFiles { + if len(f) > 0 && f[len(f)-1] != '/' { + existingChunks = append(existingChunks, "benchmark/"+f) + } + } + + if len(existingChunks) > 0 { + LOG_INFO("BENCHMARK_DELETE", "Deleting %d temporary files from previous benchmark runs", len(existingChunks)) + benchmarkRun(uploadThreads, len(existingChunks), func(threadIndex int, chunkIndex int) { + storage.DeleteFile(threadIndex, existingChunks[chunkIndex]) + }) + } + + chunks := make([][]byte, chunkCount) + chunkHashes := make([]string, chunkCount) + LOG_INFO("BENCHMARK_GENERATE", "Generating %d chunks", chunkCount) + for i := 0; i < chunkCount; i++ { + chunks[i] = make([]byte, chunkSize) + _, err = rand.Read(chunks[i]) + if err != nil { + LOG_ERROR("BENCHMARK_RAND", "Failed to generate random data: %v", err) + return false + } + hashInBytes := sha256.Sum256(chunks[i]) + chunkHashes[i] = hex.EncodeToString(hashInBytes[:]) + + } + + startTime = float64(time.Now().UnixNano()) / 1e9 + benchmarkRun(uploadThreads, chunkCount, func(threadIndex int, chunkIndex int) { + err := storage.UploadFile(threadIndex, fmt.Sprintf("benchmark/chunk%d", chunkIndex), chunks[chunkIndex]) + if err != nil { + LOG_ERROR("BENCHMARK_UPLOAD", "Failed to upload the chunk: %v", err) + return + } + }) + + runningTime = float64(time.Now().UnixNano())/1e9 - startTime + speed = int64(float64(chunkSize*chunkCount) / runningTime) + LOG_INFO("BENCHMARK_UPLOAD", "Uploaded %s bytes in %.2fs: %s/s", PrettySize(int64(chunkSize*chunkCount)), runningTime, PrettySize(speed)) + + config := CreateConfig() + + startTime = float64(time.Now().UnixNano()) / 1e9 + hashError := false + benchmarkRun(downloadThreads, chunkCount, func(threadIndex int, chunkIndex int) { + chunk := config.GetChunk() + chunk.Reset(false) + err := storage.DownloadFile(threadIndex, fmt.Sprintf("benchmark/chunk%d", chunkIndex), chunk) + if err != nil { + LOG_ERROR("BENCHMARK_DOWNLOAD", "Failed to download the chunk: %v", err) + return + } + + hashInBytes := sha256.Sum256(chunk.GetBytes()) + hash := hex.EncodeToString(hashInBytes[:]) + if hash != chunkHashes[chunkIndex] { + LOG_WARN("BENCHMARK_HASH", "Chunk %d has mismatched hashes: %s != %s", chunkIndex, chunkHashes[chunkIndex], hash) + hashError = true + } + + config.PutChunk(chunk) + }) + + runningTime = float64(time.Now().UnixNano())/1e9 - startTime + speed = int64(float64(chunkSize*chunkCount) / runningTime) + LOG_INFO("BENCHMARK_DOWNLOAD", "Downloaded %s bytes in %.2fs: %s/s", PrettySize(int64(chunkSize*chunkCount)), runningTime, PrettySize(speed)) + + if !hashError { + benchmarkRun(uploadThreads, chunkCount, func(threadIndex int, chunkIndex int) { + storage.DeleteFile(threadIndex, fmt.Sprintf("benchmark/chunk%d", chunkIndex)) + }) + LOG_INFO("BENCHMARK_DELETE", "Deleted %d temporary files from the storage", chunkCount) + } + + return true +}